DataDrive2030_Early_Learning_Predictors_Challenge.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EELfS2V88mFN"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.chdir('/content/drive/MyDrive/DataDrive2030 Early Learning Predictors Challenge')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4A3M5ukp-nzc"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8bdXcWqfJ7l7"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\n",
        "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dWKHeVZ4JpY6"
      },
      "outputs": [],
      "source": [
        "!pip install catboost -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "33eOKF3AZZ6K"
      },
      "outputs": [],
      "source": [
        "!pip install shap -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q0awp6-LJuEf"
      },
      "outputs": [],
      "source": [
        "from catboost import CatBoostRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "from xgboost import XGBRegressor\n",
        "import shap"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FkS7FKtE-_ix"
      },
      "outputs": [],
      "source": [
        "Train = pd.read_csv('Train_DD.csv')\n",
        "Test = pd.read_csv('Test_DD.csv')\n",
        "Sub = pd.read_csv('SampleSubmission.csv')\n",
        "VarD = pd.read_csv('VariableDescription.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WxAENDj4_OQ6"
      },
      "outputs": [],
      "source": [
        "Train.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P8IpuSyS_cgE"
      },
      "outputs": [],
      "source": [
        "Test.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eSYvngMzIJtB"
      },
      "outputs": [],
      "source": [
        "pd.set_option('display.max_colwidth', 100)\n",
        "VarD"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BJdTVpOrnm1a"
      },
      "outputs": [],
      "source": [
        "vard = VarD.set_index('Variable Name').T"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YFQ4tQYnVH1X"
      },
      "outputs": [],
      "source": [
        "def plot_missing_cols(df, threshold=0.8, plot=True):\n",
        "  missing_per = df.isnull().sum()/len(df)\n",
        "  missing_per_thresh = missing_per[missing_per>threshold]\n",
        "  if plot:\n",
        "    plt.figure(figsize=(20,10))\n",
        "    plt.barh(missing_per_thresh.index, missing_per_thresh.values * 100)\n",
        "    plt.xlabel('Percentage of missing values')\n",
        "    plt.title(f'Missing percentage above {threshold*100}% theshold')\n",
        "    plt.show()\n",
        "  return missing_per_thresh.index\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3AemNyp5gy8o"
      },
      "source": [
        "\n",
        "#Feature Visualization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "asUxK5f6qhI3"
      },
      "outputs": [],
      "source": [
        "def scatterplot(x,y,hue=None,length=5,width=10):\n",
        "  fig, ax = plt.subplots(figsize=(width,length))\n",
        "  sns.scatterplot(x=x,y=y,hue=hue,ax=ax)\n",
        "  ax.set_xlabel(x.name)\n",
        "  ax.set_ylabel(y.name)\n",
        "  ax.set_title(f'{x.name} vs {y.name}')\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0YvtWP6Grq7o"
      },
      "outputs": [],
      "source": [
        "def boxplot(df):\n",
        "  fig, ax = plt.subplots()\n",
        "  ax.boxplot(df)\n",
        "  ax.set_xticklabels(list(df.columns))\n",
        "  ax.set_ylabel('Value')\n",
        "  ax.set_title(f'Boxplot of {\" vs \".join(list(df.columns))}')\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4njkxSIxwi5u"
      },
      "outputs": [],
      "source": [
        "def heatmap(df, length=10, width=5, cmap='rocket'):\n",
        "  heatmap_df = df.corr()\n",
        "  fig, ax = plt.subplots(figsize=(length, width))\n",
        "  sns.heatmap(data=heatmap_df, annot=True, cmap=cmap, ax=ax)\n",
        "  ax.set_title(f'Heatmap of {\" and \".join(list(df.columns))}')\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IBPBQernMOqI"
      },
      "outputs": [],
      "source": [
        "def violinplot(x, y,hue,split=False,scale=None,length=10, width=5):\n",
        "  fig, ax = plt.subplots(figsize=(length,width))\n",
        "  sns.violinplot(x=x,y=y,hue=hue,split=split,scale=scale,ax=ax)\n",
        "  ax.set_title(f'ViolinPlot of {x.name} and {y.name}')\n",
        "  ax.set_xlabel(x.name)\n",
        "  ax.set_ylabel(y.name)\n",
        "  plt.legend(title=hue.name)\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BvTqua7BHI3N"
      },
      "outputs": [],
      "source": [
        "def barchart(x,y,length=10,width=5):\n",
        "  fig, ax = plt.subplots(figsize=(length,width))\n",
        "  sns.barplot(x=x,y=y,ax=ax)\n",
        "  ax.set_title(f'Bar Chart of {x.name} and {y.name}')\n",
        "  ax.set_xlabel(x.name)\n",
        "  ax.set_ylabel(y.name)\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rxCTpJB2JnsF"
      },
      "outputs": [],
      "source": [
        "def stack_barchart(x,y,hue,length=10,width=5):\n",
        "  fig, ax = plt.subplots(figsize=(length,width))\n",
        "  sns.barplot(x=x,y=y,hue=hue,ax=ax)\n",
        "  ax.set_title(f'Bar Chart of {x.name} and {y.name}')\n",
        "  ax.set_xlabel(x.name)\n",
        "  ax.set_ylabel(y.name)\n",
        "  plt.legend(title=hue.name)\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zo4dRk7Uf3FQ"
      },
      "outputs": [],
      "source": [
        "Train['Train'] = 'Train'\n",
        "Test['Train'] = 'Test'\n",
        "Data = pd.concat([Train, Test])\n",
        "data = Data.drop(['child_id'],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sfwvs7OcMP7_"
      },
      "outputs": [],
      "source": [
        "def targetmap(age, target):\n",
        "  if (age >= 49) and (age < 60):\n",
        "    if (target >= 0) and (target < 36.02):\n",
        "      return \"At Risk\"\n",
        "    elif (target >= 36.02) and (target < 46.32):\n",
        "      return \"Falling Behind\"\n",
        "    elif (target >= 46.32) and (target < 101):\n",
        "      return \"Achieving the Standard\"\n",
        "\n",
        "  elif (age >= 60) and (age < 70):\n",
        "    if (target >= 0) and (target < 43.24):\n",
        "      return \"At Risk\"\n",
        "    elif (target >= 43.24) and (target < 54.38):\n",
        "      return \"Falling Behind\"\n",
        "    elif (target >= 54.38) and (target < 101):\n",
        "      return \"Achieving the Standard\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5U3ojGCRO2Z1"
      },
      "outputs": [],
      "source": [
        "Train['target_cat'] = Train[['child_age','target']].apply(lambda row:targetmap(row[0],row[1]),axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z7OwlOAJm2a3"
      },
      "outputs": [],
      "source": [
        "#Checking out the Children Columns\n",
        "Train.filter(like='child').head(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##child_columns"
      ],
      "metadata": {
        "id": "bug5r_Hkz7no"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FGt4RrMmnsb8"
      },
      "outputs": [],
      "source": [
        "vard[Train.filter(like='child').columns]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XHwCHPsQEL0_"
      },
      "source": [
        "###child grant"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QOQtgVuTEKjb"
      },
      "outputs": [],
      "source": [
        "Train['child_grant'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4VXbwCVREgGG"
      },
      "outputs": [],
      "source": [
        "barchart(Train['child_grant'],Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nKZJE_yqSAny"
      },
      "outputs": [],
      "source": [
        "stack_barchart(Train['child_grant'],Train['target'],hue=Train['target_cat'])\n",
        "#Children whose primary caretaker refused the Child Support Grant have no child At Risk\n",
        "#Children whose primary caretaker refused the Child Support Grant have more children achieving the standard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z8cnIcf7TpiD"
      },
      "outputs": [],
      "source": [
        "Train['child_grant'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EjbMSdriVbl1"
      },
      "outputs": [],
      "source": [
        "Train[Train['child_grant']=='Yes']['target_cat'].value_counts()/len(Train[Train['child_grant']=='Yes'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gv-oqqT1Vlt4"
      },
      "outputs": [],
      "source": [
        "Train[Train['child_grant']=='No']['target_cat'].value_counts()/len(Train[Train['child_grant']=='No'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zmPHQpanVtd6"
      },
      "outputs": [],
      "source": [
        "Train[Train['child_grant']==\"Don't know\"]['target_cat'].value_counts()/len(Train[Train['child_grant']==\"Don't know\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "79WKKUIMJ71C"
      },
      "outputs": [],
      "source": [
        "\"\"\" Child Support Grant: 'No' - Majority of those who responded 'No' for the Child Support Grant are achieving the standard,\n",
        "which may indicate that they do not rely on the grant for financial assistance and are meeting the standard of living without it.\n",
        "A smaller percentage are at risk or falling behind, which could suggest that they may not require the grant due to their current financial situation.\n",
        "\n",
        "Child Support Grant: 'Yes' - Majority of those who responded 'Yes' for the Child Support Grant are achieving the standard,\n",
        "which may indicate that they rely on the grant for financial assistance and are able to meet the standard of living with its support.\n",
        "A significant percentage are falling behind, which could suggest that the grant may not be sufficient to lift them out of poverty entirely.\n",
        "\n",
        "Child Support Grant: 'Don't Know' - Those who responded 'Don't Know' for the Child Support Grant are relatively smaller in number compared to 'No' and 'Yes'.\n",
        "Majority of them are falling behind, which could suggest that they may not have adequate knowledge or information about the grant,\n",
        "its eligibility criteria, or how to apply for it, resulting in uncertainty about their eligibility status.\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mjkxsdtqz_Gc"
      },
      "source": [
        "###child age"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wLUkmEduw_g0"
      },
      "outputs": [],
      "source": [
        "Train['child_age'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IEf0VUipoLge"
      },
      "outputs": [],
      "source": [
        "Train[['child_age','target']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ODtMKUsesyiR"
      },
      "outputs": [],
      "source": [
        "scatterplot(Train['child_age'],Train['target'])\n",
        "boxplot(Train[['child_age','target']])\n",
        "\n",
        "#Child age doesn't seem to be correlated to the target, maybe not alone"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ory4gSU-Y_cY"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_grant'],y=Train['child_age'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dLVZLXmv1_bT"
      },
      "outputs": [],
      "source": [
        "Train['age_bin'] = pd.qcut(Train['child_age'],5,labels=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R_txy9R06Qu3"
      },
      "outputs": [],
      "source": [
        "barchart(Train['age_bin'],Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xTycZFHn6lin"
      },
      "outputs": [],
      "source": [
        "stack_barchart(Train['age_bin'],Train['target'],hue=Train['target_cat'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YSr_k1cUTgiR"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['age_bin','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['age_bin']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l3y0IyAxBcSz"
      },
      "outputs": [],
      "source": [
        "\"\"\"These insights provide information on the distribution of target categories (i.e., \"Achieving the Standard\", \"Falling Behind\", \"At Risk\") among different age groups (i.e., age bins).\n",
        "It suggests that the proportion of children in each target category varies across different age groups, with the proportion of children\n",
        "categorized as \"Achieving the Standard\" generally increasing with age, while the proportion of children categorized as \"At Risk\"\n",
        "generally decreasing with age.\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MPtyzsTR0C_5"
      },
      "source": [
        "###child enrollment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Dghdgw31mB-"
      },
      "outputs": [],
      "source": [
        "Train.filter(like='enrolment')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LFYQZDJC0WAb"
      },
      "outputs": [],
      "source": [
        "Train.filter(like='enrolment').isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DvSRE1Sr1LQR"
      },
      "outputs": [],
      "source": [
        "Train.filter(like='date')\n",
        "#It seem child_enrolment_date + child_months_enrolment gives the day the child's ELOM was taken (child_date)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qivfIB9G1-KK"
      },
      "outputs": [],
      "source": [
        "Train.filter(like='date').isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SCuytwIs1SuM"
      },
      "outputs": [],
      "source": [
        "vard[Train.filter(like='date').columns]\n",
        "#What do the dates mean? and how are they related?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q84WeE5y2rTV"
      },
      "outputs": [],
      "source": [
        "heatmap(Train[['child_months_enrolment','target']])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8HSNu7uT3G2t"
      },
      "outputs": [],
      "source": [
        "Train.corr()['child_months_enrolment'].sort_values(ascending=False)[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MYog7tnu66Qq"
      },
      "outputs": [],
      "source": [
        "Train[['count_practitioners_age_1','count_register_year_2019', 'count_register_year_2018', 'id_enumerator',\n",
        "      'count_register_year_2020', 'pri_expense_food','count_practitioners_age_0', 'count_practitioners_age_3','obs_classrooms']].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jwkrJG850zkE"
      },
      "outputs": [],
      "source": [
        "#We are missing about 69% of the Child Enrolment Dates? Should we drop the columns?\n",
        "#There doesn't seem to be an accurate way of replacing the missing dates\n",
        "#Dropping all columns with missing values up to the 80th percentile\n",
        "#There doesn't seem to be a positive correlation with months enrolment with other features, CHECK BACK LATER"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RxgltGVbJg4C"
      },
      "source": [
        "###child_years_in_programme"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7VEGRMC7JqsM"
      },
      "outputs": [],
      "source": [
        "Train['child_years_in_programme'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "afubksZRJzIM"
      },
      "outputs": [],
      "source": [
        "Train['child_years_in_programme'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CbzCXguhKG6L"
      },
      "outputs": [],
      "source": [
        "barchart(Train['child_years_in_programme'], Train['target'],10,5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jnsEuWakKUJV"
      },
      "outputs": [],
      "source": [
        "stack_barchart(Train['child_years_in_programme'],Train['target'],hue=Train['target_cat'],length=10,width=5)\n",
        "#The third year seems to have the most uneven distribution, it has the smallest distribution of the classes of child_grant\n",
        "#The second years seems to have the most even distribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w5azHNA9hIZX"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_years_in_programme','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['child_years_in_programme']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z-EhtkMyLmo8"
      },
      "outputs": [],
      "source": [
        "stack_barchart(Train['child_years_in_programme'],Train['target'],hue=Train['age_bin'],length=10,width=5)\n",
        "#The children seem to be evenly distributed by age groups in the program years"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BY0K7m6MjtHc"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_years_in_programme','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = Train.groupby(['child_years_in_programme']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fj6jXi0gny-2"
      },
      "outputs": [],
      "source": [
        "\"\"\"Age may impact program participation: The distribution of age groups may impact the number of years children spend in the programme.\n",
        "For example, children in age bin 0 (likely younger children) have higher proportions in the first and third years of the programme,\n",
        "while children in age bin 1 (slightly older children) have a higher proportion in the second year of the programme.\n",
        "This suggests that age may impact the duration of participation in the programme, with younger children potentially spending more time\n",
        "in the programme compared to older children.\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2aALNzhtl8Pt"
      },
      "source": [
        "###child_height"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZOtDGGsXjuSF"
      },
      "outputs": [],
      "source": [
        "Train['child_height'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7aIQoyyAjuXK"
      },
      "outputs": [],
      "source": [
        "Train['child_height']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CWUbhEHlzZDe"
      },
      "outputs": [],
      "source": [
        "sns.scatterplot(x=Train['child_height'],y=Train['target'],hue=Train['target_cat'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ciGuQBa6julI"
      },
      "outputs": [],
      "source": [
        "Train.corr()['child_height'].sort_values(ascending=False)[:15]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6PkIQfmwjupp"
      },
      "outputs": [],
      "source": [
        "vard['child_zha']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "47JfvbwXjuvh"
      },
      "outputs": [],
      "source": [
        "scatterplot(x=Train['child_zha'],y=Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X-_IIZFLju0E"
      },
      "outputs": [],
      "source": [
        "Train.corr()['child_zha'].sort_values(ascending=False)[:15]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ksDOzYks_Jby"
      },
      "source": [
        "###child_observe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ONSJ6eLHju4d"
      },
      "outputs": [],
      "source": [
        "Train['child_observe_attentive'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lnQ_V50cBcni"
      },
      "outputs": [],
      "source": [
        "Train['child_observe_attentive'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H7q72VGGjvAw"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_observe_attentive'],y=Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EzEzk7MhjvEe"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['target_cat'],scale='count',length=15)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CGWa9oPpAyrV"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_attentive','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['child_observe_attentive']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eIBOlhqLDzET"
      },
      "outputs": [],
      "source": [
        "stack_barchart(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['age_bin'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PO-oB7C2IvHR"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['age_bin'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JfBgLOAzJJ4U"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_attentive','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = Train.groupby(['child_observe_attentive']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YyfGgzirjvJy"
      },
      "outputs": [],
      "source": [
        "\"\"\"These insights suggest that higher levels of attentiveness (\"Often\" and \"Almost always\") are associated with a higher proportion of\n",
        "children achieving the standard, while lower levels of attentiveness (\"Sometimes\" and \"Almost never\") are associated with a higher\n",
        "proportion of children being at risk or falling behind. This implies that attentiveness may be an important factor influencing children's\n",
        "academic performance or progress.\n",
        "These insights suggest that there may be some variation in the distribution of age groups across different levels of child observance of attentiveness.\n",
        "Further analysis or statistical tests may be needed to determine if these differences are statistically significant and to understand any potential relationships\n",
        "between child attentiveness, age, and other factors.\n",
        "Based on the given insights, the age group that appears to be most attentive is age group 4, as it has the highest proportion of children\n",
        "who are observed to be attentive 'Almost always'. On the other hand, the age group that appears to be least attentive is age group 3,\n",
        "as it has the lowest proportion of children who are observed to be attentive 'Sometimes', 'Often', and 'Almost always'.\n",
        "However, it's important to note that these conclusions are based on the data provided and may not necessarily indicate a\n",
        "definitive pattern or relationship. Further analysis and statistical tests may be needed to draw more conclusive insights.\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5oRCmTs3Qjof"
      },
      "outputs": [],
      "source": [
        "stack_barchart(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['child_years_in_programme'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K1Vf0koyREn3"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['child_years_in_programme'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sn-AaTO2Rcel"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_attentive','child_years_in_programme'])[['child_years_in_programme']].count().rename(columns={'child_years_in_programme':'count'})\n",
        "total_counts = Train.groupby(['child_observe_attentive']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BgZb6ePgTYk1"
      },
      "outputs": [],
      "source": [
        "att = Train[Train['child_observe_attentive']=='Sometimes']\n",
        "att1 = Train[Train['child_observe_attentive']=='Often']\n",
        "att2 = Train[Train['child_observe_attentive']=='Almost always']\n",
        "att3 = Train[Train['child_observe_attentive']=='Almost never']\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ohbbvdR5WRSN"
      },
      "outputs": [],
      "source": [
        "violinplot(x=att['child_years_in_programme'],y=att['target'],hue=att['age_bin'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8xr7wgqzXhTO"
      },
      "outputs": [],
      "source": [
        "violinplot(x=att1['child_years_in_programme'],y=att1['target'],hue=att1['age_bin'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AdQbq_TJXhig"
      },
      "outputs": [],
      "source": [
        "violinplot(x=att2['child_years_in_programme'],y=att2['target'],hue=att2['age_bin'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FMBa6p1zjvX-"
      },
      "outputs": [],
      "source": [
        "violinplot(x=att3['child_years_in_programme'],y=att3['target'],hue=att3['age_bin'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5h4yxjLmXbMA"
      },
      "outputs": [],
      "source": [
        "grouped = att.groupby(['child_years_in_programme','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = att.groupby(['child_years_in_programme']).size()\n",
        "grouped['Sometimes'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "sm = grouped.drop('count',axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m83JXFqXjvcw"
      },
      "outputs": [],
      "source": [
        "grouped = att1.groupby(['child_years_in_programme','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = att1.groupby(['child_years_in_programme']).size()\n",
        "grouped['Often'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "oft = grouped.drop('count',axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4Zb5TA5Tjvhl"
      },
      "outputs": [],
      "source": [
        "grouped = att2.groupby(['child_years_in_programme','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = att2.groupby(['child_years_in_programme']).size()\n",
        "grouped['Almost always'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "alma = grouped.drop('count',axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "17PSm9UNjvmi"
      },
      "outputs": [],
      "source": [
        "grouped = att3.groupby(['child_years_in_programme','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = att3.groupby(['child_years_in_programme']).size()\n",
        "grouped['Almost never'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "almn = grouped.drop('count',axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hGA2U0DJjvrj"
      },
      "outputs": [],
      "source": [
        "pd.concat([sm,oft,alma,almn],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fudD19Mojvwa"
      },
      "outputs": [],
      "source": [
        "\"\"\"Older Kids tend to pay more attention than other age groups, but Older Kids are mostly only present in the First Year of the Programme\n",
        "and the category \"Don't Know\". The age group 1 pays more attentions that the other groups in the Second Year and Third Year of the Programme,\n",
        "While Group 0 which are the youngest set of children most often don't pay attention, compared to others in the Third Year of the Programme.\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M3gy36ikjv7F"
      },
      "outputs": [],
      "source": [
        "stack_barchart(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['child_grant'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "znvWiPppjv_e"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_attentive'],y=Train['target'],hue=Train['child_grant'],scale='count',length=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kl1s9IA8jwFI"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_attentive','child_grant'])[['child_grant']].count().rename(columns={'child_grant':'count'})\n",
        "total_counts = Train.groupby(['child_observe_attentive']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop('count',axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4qS0TJfbzYfE"
      },
      "outputs": [],
      "source": [
        "Train['child_observe_concentrated'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "npXOdOEDzYmu"
      },
      "outputs": [],
      "source": [
        "Train['child_observe_concentrated'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "54ObKNpi0Vca"
      },
      "outputs": [],
      "source": [
        "Train.filter(like='observe').columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qdnxPw1tzZSR"
      },
      "outputs": [],
      "source": [
        "vard[Train.filter(like='observe').columns]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_3M5UwjYB2I8"
      },
      "outputs": [],
      "source": [
        "barchart(Train['child_observe_concentrated'],Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LIzlb1vnCn5Y"
      },
      "outputs": [],
      "source": [
        "stack_barchart(Train['child_observe_concentrated'],Train['target'],hue=Train['target_cat'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4AVG5yU3C3V8"
      },
      "outputs": [],
      "source": [
        "violinplot(Train['child_observe_concentrated'],Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2nuEE90lDZKp"
      },
      "outputs": [],
      "source": [
        "\"\"\"We can Decipher from the ViolinPlot, that Children who \"Almost always\" stay concentrated on task during the activities have the highest\n",
        "possible performance Achieving the Standard and are hardly At Risk, followed by Children who \"Often\" stay concentrated on task during the activities\n",
        "those who \"Almost never\" stay concentrated have a higher chance of beeen At Risk and have a lower chance of even Acheiving the Standards similar\n",
        "to those who \"Sometimes\" stay concentrated\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-rZud8rcEu_F"
      },
      "outputs": [],
      "source": [
        "violinplot(Train['child_observe_attentive'],Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p52gagUDEvLv"
      },
      "outputs": [],
      "source": [
        "violinplot(Train['child_observe_diligent'],Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZIx2PtyTEvz2"
      },
      "outputs": [],
      "source": [
        "violinplot(Train['child_observe_interested'],Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dvIdBM1YOt_A"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_attentive', 'target_cat'])[['target_cat']].count().rename(columns={'target_cat': 'count'})\n",
        "total_counts = Train.groupby('child_observe_attentive').size()\n",
        "grouped['attentive'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "att= grouped.drop(['count'],axis=1)\n",
        "# grouped = grouped.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KwbQZJ9BR9Hx"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_interested', 'target_cat'])[['target_cat']].count().rename(columns={'target_cat': 'count'})\n",
        "total_counts = Train.groupby('child_observe_interested').size()\n",
        "grouped['interested'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "interest = grouped.drop(['count'],axis=1)\n",
        "# grouped = grouped.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pmvCXePBR9Xz"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_concentrated', 'target_cat'])[['target_cat']].count().rename(columns={'target_cat': 'count'})\n",
        "total_counts = Train.groupby('child_observe_concentrated').size()\n",
        "grouped['concentrated'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "conc = grouped.drop(['count'],axis=1)\n",
        "# grouped = grouped.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-egv9oOJR9jt"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_observe_diligent', 'target_cat'])[['target_cat']].count().rename(columns={'target_cat': 'count'})\n",
        "total_counts = Train.groupby('child_observe_diligent').size()\n",
        "grouped['diligent'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "dilig = grouped.drop(['count'],axis=1)\n",
        "# grouped = grouped.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HRLvNTT1Sjg_"
      },
      "outputs": [],
      "source": [
        "pd.concat([att,conc,dilig,interest],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2G8PcrS3dx1r"
      },
      "outputs": [],
      "source": [
        "#child_observe_diligent > child_observe_attentive > child_observe_interested > child_observe_concentrated"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fadln11ue3vR"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_concentrated'],y=Train['child_age'],hue=Train['age_bin'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RuSpEDoeKmbR"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_attentive'],y=Train['child_age'],hue=Train['age_bin'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3QFR_FYwKmoF"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_diligent'],y=Train['child_age'],hue=Train['age_bin'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9QkwwO-5Kmzg"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_interested'],y=Train['child_age'],hue=Train['age_bin'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f1ha20-jK-_V"
      },
      "outputs": [],
      "source": [
        "\"\"\"In all different categories of observation, it seems like age_group 4 are more dominant in the Almost Always category, and\n",
        "age_group 0 are more dominant in the Almost never category, this shows that older aged children tend to show more interest, diligence,\n",
        "concentration and attention, while younger aged children seem to be more carefree\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6F8H9eDShecH"
      },
      "outputs": [],
      "source": [
        "sns.stripplot(x=Train['child_observe_total'],y=Train['target'],hue=Train['target_cat'],jitter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2g_uq-0xjiBp"
      },
      "outputs": [],
      "source": [
        "sns.stripplot(x=Train['child_observe_total'],y=Train['child_age'],hue=Train['target_cat'],jitter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5TDL3yvZkMLf"
      },
      "outputs": [],
      "source": [
        "sns.stripplot(x=Train['child_observe_total'],y=Train['child_height'],hue=Train['target_cat'],jitter=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7wUKnbuFgh-R"
      },
      "source": [
        "###child_gender"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tBJ8Rwu7kf35"
      },
      "outputs": [],
      "source": [
        "Train['child_gender'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CvQkmiQvkqpz"
      },
      "outputs": [],
      "source": [
        "Train['child_gender'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gS0UAOLmkxa-"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_gender'],y=Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pS_5_0dWk6Og"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_gender'],y=Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RSePE83-lZIe"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_gender','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['child_gender']).size()\n",
        "grouped['normalized_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wXzUZorVlQNR"
      },
      "outputs": [],
      "source": [
        "\"\"\"It seems like Female Kids are more likely to Achieve the standard than Male Kids, but Male kids have a higher chance to be At Risk and Falling Behind\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xJs1E1PGnJwN"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_diligent'],y=Train['target'],hue=Train['child_gender'],split=True,scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q-K9z21eoDIJ"
      },
      "outputs": [],
      "source": [
        "\"\"\"The Female gender has a more spread out distribution than the male counterpart, for the attention levels Almost always and Often, and have\n",
        "a smaller distribution than their male counterparts for the attention levels Sometimes and Often\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f6UzXWxTonxJ"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_observe_diligent'],y=Train['child_age'],hue=Train['child_gender'],split=True,scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KryZ-Gs-phLj"
      },
      "outputs": [],
      "source": [
        "\"\"\"\"The Female gender has a more spread out distribution than the male counterpart, for the attention levels Almost always and Often, and have\n",
        "a smaller distribution than their male counterparts for the attention levels Sometimes and Often\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DpaWYXDHkgkX"
      },
      "source": [
        "###child_dob"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7kx2ozGekiWx"
      },
      "outputs": [],
      "source": [
        "Train['child_dob'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ipOdEKLa7KPW"
      },
      "outputs": [],
      "source": [
        "Train['Year_dob'] = Train['child_dob'].str.split('-',expand=True)[0].astype(float)\n",
        "Train['Month_dob'] = Train['child_dob'].str.split('-',expand=True)[1].astype(float)\n",
        "Train['Day_dob'] = Train['child_dob'].str.split('-',expand=True)[2].astype(float)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fHQuOzOp7OFi"
      },
      "outputs": [],
      "source": [
        "Train['Year_dob'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WHnfoeK_8jWn"
      },
      "outputs": [],
      "source": [
        "Train['Month_dob'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7kORqhnG8suI"
      },
      "outputs": [],
      "source": [
        "Train['Day_dob'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ELYOg7Gm9A4u"
      },
      "outputs": [],
      "source": [
        "heatmap(Train[['Year_dob','Month_dob','Day_dob','child_age','age_bin']])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AW3qSZ_Q82qZ"
      },
      "outputs": [],
      "source": [
        "barchart(Train['Year_dob'],Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kioAL_Ca9fRs"
      },
      "outputs": [],
      "source": [
        "violinplot(Train['Year_dob'],Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nU44gNzg9yRz"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['Year_dob','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['Year_dob']).size()\n",
        "grouped['normalized_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JAu12Yqwv1BI"
      },
      "outputs": [],
      "source": [
        "grouped = grouped.reset_index()\n",
        "sns.catplot(data=grouped,x='Year_dob',y='normalized_count',hue='target_cat',kind='point')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zu6VW8GnAGNr"
      },
      "outputs": [],
      "source": [
        "\"\"\"There seems to be an upward and downward trend among the Years, the target performance improved from the year 2015 - 2016 the became worse in the year 2017\n",
        "the become better for those born in the year 2018, why is there a downwards trend between children born in 2016-2017, and why is there an upwards trend\n",
        "between those born in 2017-2018, same happened in 2015-2016\"\"\"\n",
        "pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mF5D_WKyA_zX"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['Year_dob','Month_dob','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['Year_dob','Month_dob']).size()\n",
        "xy=pd.DataFrame(total_counts,columns=['size'])\n",
        "xyz = pd.merge(grouped.reset_index(), xy.reset_index(), left_on=['Year_dob', 'Month_dob'], right_on=['Year_dob', 'Month_dob'])\n",
        "xyz['count'] = xyz['count']/xyz['size']\n",
        "xyz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2yujnJjjsWDF"
      },
      "outputs": [],
      "source": [
        "sns.catplot(data=xyz,x='Month_dob',y='count',col='Year_dob',hue='target_cat',kind='point',height=6)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VotwdkgB_y42"
      },
      "outputs": [],
      "source": [
        "#There doesn't seem to be a pattern, just a random distribution"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R6kKorlpkipY"
      },
      "source": [
        "###child_zha"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZvoH0FU9AFAA"
      },
      "outputs": [],
      "source": [
        "Train['child_zha'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p7pnezKWAjb2"
      },
      "outputs": [],
      "source": [
        "sns.scatterplot(x=Train['child_zha'],y=Train['target'],hue=Train['target_cat'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zRB3kdgyCOLp"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_gender'],y=Train['child_zha'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eYLee7sGCOTE"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['age_bin'],y=Train['child_zha'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XwW4G0h3D158"
      },
      "source": [
        "###child_stunted"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "alsxekyVD_8M"
      },
      "outputs": [],
      "source": [
        "Train['child_stunted'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AvLakpy0EACr"
      },
      "outputs": [],
      "source": [
        "Train['child_stunted'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cKTkgTZVEAMK"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_stunted'],y=Train['target'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OVtfzTUDEAvp"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_stunted'],y=Train['target'],hue=Train['target_cat'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ryVrKNLtE1yu"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_stunted','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_count = Train.groupby(['child_stunted']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_count[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YyccRWHEGD-Y"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_stunted'],y=Train['child_height'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RmWNWRU1GQKF"
      },
      "outputs": [],
      "source": [
        "barchart(x=Train['child_stunted'],y=Train['child_zha'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kQye1oNrGr4E"
      },
      "outputs": [],
      "source": [
        "violinplot(x=Train['child_stunted'],y=Train['child_age'],hue=Train['age_bin'],scale='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s-kHWrYBIjMb"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_stunted','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = Train.groupby(['child_stunted']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_count[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "31s-2DeRD2Gq"
      },
      "source": [
        "####child_attends"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cSlE_iiXJSvV"
      },
      "outputs": [],
      "source": [
        "Train['child_attends'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xv4KyxhED2QG"
      },
      "source": [
        "####child_attendance"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y1K9eUZDJfNK"
      },
      "outputs": [],
      "source": [
        "Train['child_attendance'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVDNPNRLD2Zu"
      },
      "source": [
        "####child_languages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PMaVBfxoJyOS"
      },
      "outputs": [],
      "source": [
        "Train['child_languages'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8e29TT76D2m8"
      },
      "source": [
        "###child_age_group"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fYWiVGFsJ--s"
      },
      "outputs": [],
      "source": [
        "Train['child_age_group'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bq3_Cn2nKFBl"
      },
      "outputs": [],
      "source": [
        "Train['child_age_group'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uWViSBgXKm2l"
      },
      "outputs": [],
      "source": [
        "Train.groupby(['child_age_group','age_bin'])[['age_bin']].count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D_3FWp3yMj7z"
      },
      "outputs": [],
      "source": [
        "grouped = Train.groupby(['child_age_group','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['child_age_group']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LVzReCe-D2x5"
      },
      "source": [
        "####pri_children_4_6_years"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bpWCd0pwNxNG"
      },
      "outputs": [],
      "source": [
        "Train['pri_children_4_6_years'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OnKMz5pZD27X"
      },
      "source": [
        "####obs_toilets_children\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iugUEhvCOoc3"
      },
      "outputs": [],
      "source": [
        "Train['obs_toilets_children'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CATWNhqdPhen"
      },
      "outputs": [],
      "source": [
        "Train['obs_toilets_children'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0FZti6dtD3EW"
      },
      "source": [
        "###count_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Il_P-SuGO4xB"
      },
      "outputs": [],
      "source": [
        "Train['count_children_present'].isnull().sum()/len(Train)\n",
        "Test['count_children_present'].isnull().sum()/len(Test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LICi5RUEUN1a"
      },
      "outputs": [],
      "source": [
        "count_cols = Train.filter(like='count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "j6ZRG7TOUzER"
      },
      "outputs": [],
      "source": [
        "missing_cols=plot_missing_cols(count_cols,plot=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4YeP2zuyV4eD"
      },
      "outputs": [],
      "source": [
        "count_cols = count_cols.drop(missing_cols,axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#####count_staff"
      ],
      "metadata": {
        "id": "4TivrDKi2p7m"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TEaIjbTdWGET"
      },
      "outputs": [],
      "source": [
        "vard[count_cols.filter(like='staff').columns]\n",
        "#Count_staff_all and Count_staff_gender are the same thing, Count_staff_salary is the total number of paid staffs\n",
        "#count_staff_contract and\tcount_staff_time are the same, count_staff_time_full"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Train['count_staff_skills'] = Train['count_staff_qual_skills']*0.2+Train['count_staff_qual_nqf4_5']*0.2+Train['count_staff_qual_nqf6_9']*0.6"
      ],
      "metadata": {
        "id": "Zon6-OxckMrn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_grant'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "#Staff Qualifications: The count of staff with skills, NQF Level 4-5 qualification, and NQF Level 6-9 qualification has an impact on the\n",
        "#child_grant category. The 'count_staff_skills' metric, which is a weighted combination of these qualifications, is highest for the\n",
        "#'No' category of child_grant, indicating that programs with staff having higher qualifications tend to have fewer children receiving the\n",
        "#Child Support Grant.\n",
        "\n",
        "#Staff Salary: The average count of staff salary paid is highest for the 'No' category of child_grant, indicating that programs where\n",
        "#children do not receive the Child Support Grant tend to have higher staff salaries."
      ],
      "metadata": {
        "id": "VdjfiYZjpc-S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['age_bin'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "#Overall, the data suggests that as the age group category increases, the staff qualifications, staff salary, proportion of female staff\n",
        "#members, and target outcome tend to increase as well. This could indicate that programs targeting older children may have different\n",
        "#staffing and operational requirements compared to programs targeting younger children."
      ],
      "metadata": {
        "id": "CNtbJXwusdir"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['Year_dob'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()"
      ],
      "metadata": {
        "id": "G0q0kHVAtQz9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_years_in_programme'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "#Staff Qualifications: The average count of staff skills ('count_staff_skills') tends to increase slightly from the 1st year to the 3rd\n",
        "#year in the program, with the highest average observed in the 3rd year. This suggests that staff qualifications may improve over time as\n",
        "#they gain more experience in the program.\n",
        "\n",
        "#Staff Salary: The average count of staff salary paid ('count_staff_salary_paid') also tends to increase from the 1st year to the\n",
        "#3rd year in the program, indicating that staff may receive higher salaries as they progress in their tenure.\n",
        "\n",
        "#Gender Distribution: The average count of female staff members ('count_staff_gender_female') does not show a significant trend\n",
        "#across different years of enrollment in the program, as the values are relatively similar.\n",
        "\n",
        "#Target Outcome: The average target outcome, which is not defined in the given data, shows a slight fluctuation across different\n",
        "#years of enrollment in the program, with the highest average observed in the \"Do Not Know\" category."
      ],
      "metadata": {
        "id": "cAP8NrxosdvA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "aa = Train.groupby(['child_observe_attentive'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "ab = Train.groupby(['child_observe_interested'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "ac = Train.groupby(['child_observe_concentrated'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "ad = Train.groupby(['child_observe_diligent'])[['count_staff_skills','count_staff_salary_paid','count_staff_gender_female','target']].mean()\n",
        "\n",
        "ab = aa.join(ab,lsuffix='_aa',rsuffix='_ab')\n",
        "ad = ac.join(ad,lsuffix='_ac',rsuffix='_ad')\n",
        "at = ab.join(ad)\n",
        "at['average_staff_skills'] = np.mean(at.filter(like='skills'),axis=1).values\n",
        "at['average_staff_salary_paid'] = np.mean(at.filter(like='salary'),axis=1).values\n",
        "at['average_staff_gender_female'] = np.mean(at.filter(like='gender'),axis=1).values\n",
        "at['average_target'] = np.mean(at.filter(like='target'),axis=1).values\n",
        "\n",
        "at = at.filter(like='average')\n",
        "at\n",
        "\n",
        "#Overall, the data suggests that child observation levels may have an impact on staff qualifications, staff salary, and target outcome."
      ],
      "metadata": {
        "id": "zYluQy7Osd6T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sns.catplot(data=Train,x='count_staff_salary_paid',y='target',kind='boxen',height=10)"
      ],
      "metadata": {
        "id": "KwgmY2THYPxf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nRveFQ2iFOu-"
      },
      "outputs": [],
      "source": [
        "sns.catplot(data=Train,x='count_staff_gender_female',y='target',kind='boxen',height=10)\n",
        "#A lot of Outliers present in both count_staff_gender_female and target variable, is there a reason why?"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#####count_register"
      ],
      "metadata": {
        "id": "vZ-AFymp2y8P"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LkGABdWEAgZx"
      },
      "outputs": [],
      "source": [
        "vard[count_cols.filter(like='register').columns]\n",
        "#count_register_all and count_register_gender are the same"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "count_cols.filter(like='race')"
      ],
      "metadata": {
        "id": "3SsdgRdJ1zpX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "count_cols.filter(like='race').describe()"
      ],
      "metadata": {
        "id": "Rr1KcZca2DG-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['Year_dob'])[['count_register_race_african','count_register_race_coloured',\n",
        "                             'count_register_race','target']].mean()"
      ],
      "metadata": {
        "id": "Khre4afP1zxb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_grant'])[['count_register_race_african','count_register_race_coloured',\n",
        "                             'count_register_race','target']].mean()\n",
        "#Child Grant and Count Register: There are differences in the average counts of children registered in the ECD Programme\n",
        "#(\"count_register_race\") based on whether the child is receiving a grant or not. Specifically, children who are reported as\n",
        "#\"Don't know\" or \"Refuse\" regarding their grant status tend to have higher average counts of African/Black children\n",
        "#(\"count_register_race_african\") compared to children who are not receiving a grant (\"No\") or are receiving a grant (\"Yes\").\n",
        "#This suggests that there may be some association between grant status and the counts of African/Black children in the ECD Programme."
      ],
      "metadata": {
        "id": "VW9b537a28x7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"Count of Registered Children: The average counts of African/Black children (\"count_register_race_african\"), Coloured children\n",
        "(\"count_register_race_coloured\"), and the overall count of registered children (\"count_register_race\") show some variability across\n",
        "different age bins. Specifically, the counts tend to decrease from age bin 0 to age bin 4. This suggests that there may be some\n",
        " differences in the enrollment or registration of children from different racial/ethnic backgrounds based on their age, with\n",
        " lower counts observed in higher age bins.\"\"\"\n",
        "\n",
        "Train.groupby(['age_bin'])[['count_register_race_african','count_register_race_coloured',\n",
        "                             'count_register_race','target']].mean()\n"
      ],
      "metadata": {
        "id": "RsOONIhx28-v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_years_in_programme'])[['count_register_race_african','count_register_race_coloured',\n",
        "                             'count_register_race','target']].mean()\n",
        "#The number of African children increases over the years, while the number of coloured children decreases over the years"
      ],
      "metadata": {
        "id": "3rjCq3rM29Md"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "aa = Train.groupby(['child_observe_attentive'])[['count_register_race_african','count_register_race_coloured','count_register_race','target']].mean()\n",
        "ab = Train.groupby(['child_observe_interested'])[['count_register_race_african','count_register_race_coloured','count_register_race','target']].mean()\n",
        "ac = Train.groupby(['child_observe_concentrated'])[['count_register_race_african','count_register_race_coloured','count_register_race','target']].mean()\n",
        "ad = Train.groupby(['child_observe_diligent'])[['count_register_race_african','count_register_race_coloured','count_register_race','target']].mean()\n",
        "\n",
        "ab = aa.join(ab,lsuffix='_aa',rsuffix='_ab')\n",
        "ad = ac.join(ad,lsuffix='_ac',rsuffix='_ad')\n",
        "at = ab.join(ad)\n",
        "at['average_register_race_african'] = np.mean(at.filter(like='african'),axis=1).values\n",
        "at['average_register_race_coloured'] = np.mean(at.filter(like='coloured'),axis=1).values\n",
        "at['average_target'] = np.mean(at.filter(like='target'),axis=1).values\n",
        "\n",
        "at = at.filter(like='average')\n",
        "at"
      ],
      "metadata": {
        "id": "uS-m6qW_295d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['Black-Coloured-Ratio'] = Train['count_register_race'] / Train['count_register_race_african']"
      ],
      "metadata": {
        "id": "kryJrS19FlUU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['Black-Coloured-Ratio'] = Train['Black-Coloured-Ratio'].replace(np.inf, 0)"
      ],
      "metadata": {
        "id": "BH0WDLodGC6M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "count_cols.filter(like='register_year').describe()"
      ],
      "metadata": {
        "id": "FQ8PrK1wVsjM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['count_register_year_total'] = count_cols.filter(like='register_year').drop(['count_register_year_grader','count_register_year_school'],axis=1).sum(axis=1)"
      ],
      "metadata": {
        "id": "qDbncit2hVzq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['Year_dob'])[['count_register_all','count_register_year_grader',\n",
        "                             'count_register_year_school','target']].mean()"
      ],
      "metadata": {
        "id": "yCOs2ruYYRMD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_grant'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()"
      ],
      "metadata": {
        "id": "1hYTGg9hgStp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['age_bin'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()"
      ],
      "metadata": {
        "id": "CQkdGgi9tgHM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_years_in_programme'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()\n",
        "#Participation in the ECD Programme: Children who have been in the programme for longer durations (2nd and 3rd year) tend to have\n",
        "#slightly higher average counts of children registered in the ECD Programme (\"count_register_all\") compared to children in their\n",
        "#1st year in the programme. This suggests that there may be an increasing trend in participation in the ECD Programme over time.\n",
        "\n",
        "#Grade R Participation: Children who have been in the programme for longer durations (2nd and 3rd year) tend to have slightly\n",
        "#higher average counts of children registered in Grade R (\"count_register_year_grader\") compared to children in their 1st year in\n",
        "#the programme. This indicates that children who have been in the programme for multiple years are more likely to progress to Grade R,\n",
        "#which is a higher level of early childhood education.\n",
        "\n",
        "#School Age Participation: Children who have been in the programme for longer durations (2nd and 3rd year) tend to have slightly higher\n",
        "#average counts of children registered in school age (\"count_register_year_school\") compared to children in their 1st year in the\n",
        "#programme. This suggests that children who have been in the programme for multiple years are more likely to reach the school age and\n",
        "#continue their education."
      ],
      "metadata": {
        "id": "yZ4m1olzvwLT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "aa = Train.groupby(['child_observe_attentive'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()\n",
        "ab = Train.groupby(['child_observe_interested'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()\n",
        "ac = Train.groupby(['child_observe_concentrated'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()\n",
        "ad = Train.groupby(['child_observe_diligent'])[['count_register_all','count_register_year_grader','count_register_year_school','target']].mean()\n",
        "\n",
        "ab = aa.join(ab,lsuffix='_aa',rsuffix='_ab')\n",
        "ad = ac.join(ad,lsuffix='_ac',rsuffix='_ad')\n",
        "at = ab.join(ad)\n",
        "at['average_register_all'] = np.mean(at.filter(like='all'),axis=1).values\n",
        "at['average_year_grader'] = np.mean(at.filter(like='grader'),axis=1).values\n",
        "at['average_year_school'] = np.mean(at.filter(like='school'),axis=1).values\n",
        "at['average_target'] = np.mean(at.filter(like='target'),axis=1).values\n",
        "\n",
        "at = at.filter(like='average')\n",
        "at"
      ],
      "metadata": {
        "id": "JN2o8aPjxbuc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "heatmap(Train[['count_register_all','count_register_gender_female','count_register_gender_male','Year_dob','Month_dob','target']])"
      ],
      "metadata": {
        "id": "oFhai9aZD_Fs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#####count_children"
      ],
      "metadata": {
        "id": "isAsIwQrKVqF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3jjJow68AmsT"
      },
      "outputs": [],
      "source": [
        "vard[count_cols.filter(like='children').columns]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lyxY2krJXZ5L"
      },
      "outputs": [],
      "source": [
        "Train[['count_children_present','count_register_all','count_children_attendance','count_children_precovid']]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['count_children_present','count_register_all','count_children_attendance','count_children_precovid','count_toilets_children']].describe()"
      ],
      "metadata": {
        "id": "7ybzgpirWClm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "heatmap(Train[['count_children_present','count_register_all','count_children_attendance','count_children_precovid']])"
      ],
      "metadata": {
        "id": "GCG8ML1NWCsw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['target_cat'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n"
      ],
      "metadata": {
        "id": "bbXwe8o3W_sf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_grant'])[['count_children_present','count_children_attendance','count_children_precovid']]"
      ],
      "metadata": {
        "id": "BSr_lEGFbdn1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8HmbdX9APNCi"
      },
      "outputs": [],
      "source": [
        "Train.groupby(['child_grant'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['age_bin'],sort=True)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()"
      ],
      "metadata": {
        "id": "BALZ6we9bFa_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['child_years_in_programme'],sort=True)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()"
      ],
      "metadata": {
        "id": "pLep5jLmbFO9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "aa = Train.groupby(['child_observe_attentive'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n",
        "ab = Train.groupby(['child_observe_interested'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n",
        "ac = Train.groupby(['child_observe_concentrated'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n",
        "ad = Train.groupby(['child_observe_diligent'],sort=False)[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children',\n",
        "                                      'Black-Coloured-Ratio','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()\n",
        "\n",
        "ab = aa.join(ab,lsuffix='_aa',rsuffix='_ab')\n",
        "ad = ac.join(ad,lsuffix='_ac',rsuffix='_ad')\n",
        "at = ab.join(ad)\n",
        "at['average_children_present'] = np.mean(at.filter(like='present'),axis=1).values\n",
        "at['average_children_attendance'] = np.mean(at.filter(like='attendance'),axis=1).values\n",
        "at['average_children_precovid'] = np.mean(at.filter(like='precovid'),axis=1).values\n",
        "at['average_toilets_children'] = np.mean(at.filter(like='toilets'),axis=1).values\n",
        "at['average_Black-Coloured-Ratio'] = np.mean(at.filter(like='Ratio'),axis=1).values\n",
        "at['average_staff_salary_paid'] = np.mean(at.filter(like='salary'),axis=1).values\n",
        "at['average_staff_skills'] = np.mean(at.filter(like='skills'),axis=1).values\n",
        "at['average_staff_gender_female'] = np.mean(at.filter(like='gender'),axis=1).values\n",
        "\n",
        "at = at.filter(like='average')\n",
        "at"
      ],
      "metadata": {
        "id": "YMFKZPbDbFCa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cn5Za1u2D3qP"
      },
      "source": [
        "###language_child"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gbbAcYnvPYkR"
      },
      "outputs": [],
      "source": [
        "Train['language_child'].isnull().sum()/len(Train)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "barchart(Train['language_child'],Train['target'])"
      ],
      "metadata": {
        "id": "DIOqiYtd3W-e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "violinplot(x=Train['language_child'],y=Train['target'],hue=Train['target_cat'],scale='count',length=20,width=10)"
      ],
      "metadata": {
        "id": "z8oSeBSJ4N5P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['language_child'].value_counts()"
      ],
      "metadata": {
        "id": "UabQ43bn1hOM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['language_child','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['language_child']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.drop(['count'],axis=1,inplace=True)\n",
        "grouped"
      ],
      "metadata": {
        "id": "cr7PuNLE5B5K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['language_child'])[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children']].mean()"
      ],
      "metadata": {
        "id": "Vo9lTFco79HZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Children who speak English, siSwati and Tshivenda are the majority in How many children are present at the ECD Programme today? (\"count_children_present\") and How many children usually attend the ECD Programme, on average? (\"count_children_attendance\"), but in the How many children usually attended the ECD programme before the start of COVID on average\" Tshivenda had the highest count, followed by the English and the Sesotho languages, the Amount of Children from Tshivenda and Sesotho dropped. Afrikaans, isiXhosa and Xitsonga have the majority in How many childrens toilets are there? why?"
      ],
      "metadata": {
        "id": "FFu1c_d89GZc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['language_child'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female','count_staff_skills','count_staff_salary_paid']].mean()"
      ],
      "metadata": {
        "id": "UahGo7Oj_3OQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "siSwati, Sesotho, Xitsonga have the highest number of registered african race, Afrikaans and English have the highest number of registered coloured race, English, Afrikaans and Xitsonga have the highest number of female staffs, Afrikaans, English and siSwati have the staff with the highest skills, English, Sesotho and Xitsongs pay the staff more"
      ],
      "metadata": {
        "id": "vEgriQB2AASk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_grant','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_grant']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')\n",
        "#Afrikaans and English are the majority who said No to the child_grant\n",
        "#isiXhosa and isiZulu are the majority who said Yes to the child_grant"
      ],
      "metadata": {
        "id": "uNojolBF_3YG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['age_bin','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['age_bin']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')\n",
        "#Setswana, isiZulu, isiXhosa are the major youngest children in the ECD_Programme\n",
        "#Setswana, isiZulu, isiXhosa are the major children in the age_bin 1 in the ECD_Programme\n",
        "#Setswana, isiZulu, isiXhosa are the major children in the age_bin 2 in the ECD_Programme\n",
        "#isiZulu, isiXhosa, Afrikaans are the major children in the age_bin 3 in the ECD_Programme\n",
        "#Afrikaans, isiXhosa, isiZulu are the major children in the age_bin 3 in the ECD_Programme\n"
      ],
      "metadata": {
        "id": "995CGljOQAcq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_years_in_programme','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_years_in_programme']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')\n",
        "#IsiZulu, Setswana, IsiXhosa are the major children in the 1st year of the programme\n",
        "#Setswana, IsiZulu, IsiXhosa are the major children in the 2nd year of the programme\n",
        "#IsiXhosa, IsiZulu, Setswana are the major children in the 3rd year of the programme"
      ],
      "metadata": {
        "id": "z8kfUo0RRsm9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_gender','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_gender']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')\n",
        "#IziXhosa, IziZulu and Setswana all the way"
      ],
      "metadata": {
        "id": "BNmFkggGPApW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_observe_attentive','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_observe_attentive']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "iD5dyuiwSydT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_observe_interested','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_observe_interested']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "vwRkBOD0TE4F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_observe_concentrated','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_observe_concentrated']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "ac = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "kqry0frnTFGs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['child_observe_diligent','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['child_observe_diligent']).size()\n",
        "grouped['norm_count'] = grouped['count'] / total_counts[grouped.index.get_level_values(0)].values\n",
        "ad = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "bnw1aCxbTFi7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bMB1uMZ1D3z-"
      },
      "source": [
        "##ID"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Train['prov_best'].value_counts()"
      ],
      "metadata": {
        "id": "352hLbuvpDz-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','target_cat'])[['target_cat']].count().rename(columns={'target_cat':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "gl5Cl42MqWbK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "    Provinces with higher proportions of children achieving the standard: Provinces such as Mpumalanga (0.585477), Western Cape (0.552212), and Gauteng (0.518057) have higher proportions of children achieving the standard compared to other provinces. This suggests that these provinces may have better educational outcomes for children, or better access to resources and opportunities that contribute to higher achievement levels.\n",
        "\n",
        "    Provinces with higher proportions of children falling behind or at risk: Provinces such as Limpopo (0.317443 falling behind, 0.309859 at risk), Northern Cape (0.268139 falling behind, 0.364353 at risk), and Free State (0.257206 falling behind, 0.379157 at risk) have higher proportions of children falling behind or at risk compared to other provinces. This indicates that these provinces may have challenges in terms of educational outcomes or access to resources, which result in higher proportions of children facing difficulties in meeting the standard.\n",
        "\n",
        "    Provinces with similar proportions across target categories: Provinces such as Eastern Cape (similar proportions across all target categories), KwaZulu-Natal (similar proportions across all target categories), North West (similar proportions across all target categories), and Unknown (similar proportions across all target categories) have relatively similar proportions of children across different target categories. This suggests that the educational outcomes for children in these provinces may be relatively balanced or similar across the three target categories.\n",
        "\n",
        "    Variation in proportions across provinces: The data also reveals that there is variation in the proportions of children in different target categories across provinces, with some provinces having higher or lower proportions compared to others. This highlights the importance of considering provincial-level differences when analyzing educational outcomes and developing interventions or policies to address the needs of children in different regions."
      ],
      "metadata": {
        "id": "HlpofvN3uM2M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','language_child'])[['language_child']].count().rename(columns={'language_child':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "9ZTk8EcAu-aY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "    Language dominance: The data shows that English is the dominant language in most provinces, except for Eastern Cape where isiXhosa is the most commonly spoken language. This suggests that English is widely used and spoken across different provinces, which could be due to historical and cultural factors, as well as its status as a global language.\n",
        "\n",
        "    Regional language variations: The data also reveals that certain provinces have a higher prevalence of specific regional languages. For example, Sesotho is prominent in Free State and Northern Cape, while Setswana is dominant in North West province. This suggests that there are regional language variations within South Africa, which may be influenced by factors such as geography, history, and cultural heritage.\n",
        "\n",
        "    Language preservation: The data shows that some provinces have a higher proportion of indigenous languages, such as isiXhosa in Eastern Cape and isiZulu in KwaZulu-Natal, which indicates the efforts to preserve and maintain local languages. This highlights the importance of cultural preservation and recognition of linguistic diversity in South Africa.\n",
        "\n",
        "    Language accessibility: The data also reveals that there are variations in language accessibility across provinces, with some languages being less commonly spoken or represented. For example, Afrikaans is less prevalent in some provinces compared to others, which could indicate differences in language accessibility and usage patterns based on historical, cultural, and socio-economic factors.\n",
        "\n",
        "    Normative language use: The data provides insights into the normative language use in different provinces, with some languages being more commonly used in certain contexts or settings. For example, Afrikaans is more dominant in Western Cape, while Sepedi and Setswana are more prevalent in Mpumalanga and North West provinces, respectively. This suggests that language use is influenced by various factors such as socio-economic status, education, and cultural norms.\n",
        "\n",
        "    Unknown language usage: The data also reveals that there are instances of \"Unknown\" language usage, particularly in the province of Gauteng. This could indicate challenges in accurately capturing and categorizing language data, and the need for further investigation and clarification to better understand language dynamics in the region.\n",
        "\n",
        "In conclusion, the data provides valuable insights into language usage patterns in different provinces of South Africa, highlighting the dominance of English, regional language variations, efforts towards language preservation, variations in language accessibility, normative language use, and challenges in capturing accurate language data. These insights can be useful for policymakers, educators, and researchers in understanding language dynamics in South Africa and addressing issues related to language policy, cultural preservation, and linguistic diversity."
      ],
      "metadata": {
        "id": "NC06Ke4kzCmD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','age_bin'])[['age_bin']].count().rename(columns={'age_bin':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "H_rolJbau-jI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "    Provinces with higher percentages of young children: Provinces like Free State, Limpopo, Mpumalanga, and Northern Cape have a higher percentage of young children in the 0-1 age group, which could indicate higher birth rates or larger populations of young families in these provinces. This may have implications for healthcare services, early childhood education, and other services targeted towards young children and their families.\n",
        "\n",
        "    Consistent age distribution patterns: Provinces like Gauteng and KwaZulu-Natal have similar normalized counts across different age groups, indicating a relatively consistent age distribution pattern. This may suggest that these provinces have a stable population structure and may require similar policy interventions and resource allocations for different age groups.\n",
        "\n",
        "    Variation in age distribution within provinces: Provinces like Free State and Northern Cape have more variation in normalized counts across different age groups, indicating differences in age demographics within the province. This may suggest the need for tailored policies and services to address the specific needs of different age groups within these provinces.\n",
        "\n",
        "    Data gaps and missing information: The \"Unknown\" category with a significant percentage of people in the 0-1 age group may indicate data gaps or missing information in certain provinces. This highlights the need for further investigation and validation of data quality and reliability, and cautious interpretation of insights based on this data."
      ],
      "metadata": {
        "id": "yefYfoEj2kUa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','child_observe_attentive'])[['child_observe_attentive']].count().rename(columns={'child_observe_attentive':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "at = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "riYKO9Mdu-pm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','child_observe_concentrated'])[['child_observe_concentrated']].count().rename(columns={'child_observe_concentrated':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ac = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "wCYo2M2L393n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','child_observe_interested'])[['child_observe_interested']].count().rename(columns={'child_observe_interested':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ai = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "tckilcwD3-MU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','child_observe_diligent'])[['child_observe_diligent']].count().rename(columns={'child_observe_diligent':'count'})\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['norm_count'] = grouped['count']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ad = grouped.filter(like='norm')"
      ],
      "metadata": {
        "id": "wxzpeYPBu-vb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "at['child_observe_concentrated'] = ac['norm_count']\n",
        "at['child_observe_interested'] = ai['norm_count']\n",
        "at['child_observe_diligent'] = ad['norm_count']\n",
        "at = at.rename(columns={'norm_count':'child_observe_att'})"
      ],
      "metadata": {
        "id": "3n9D_5ll4xMg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "at['average_observe_levels'] = at.filter(like='observe').mean(axis=1)\n",
        "at.filter(like='average')"
      ],
      "metadata": {
        "id": "t6Jv6G6o5DT0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "    Regional Disparities: The data suggests that there may be regional disparities in the frequency of child observation levels in South Africa, with some provinces showing higher levels of child observation compared to others. This could be indicative of differences in child care practices, access to resources, and socio-economic conditions across different regions in the country.\n",
        "\n",
        "    Policy Interventions: The data could be used to support the need for targeted policy interventions to improve child observation levels, particularly in provinces with lower average observation levels. For example, provinces with lower levels of child observation could be prioritized for interventions such as parenting programs, early childhood development initiatives, and caregiver support services to promote positive child observation behaviors.\n",
        "\n",
        "    Further Research: The data could also highlight the need for further research to explore the underlying factors that contribute to the observed variations in child observation levels across different provinces in South Africa. Factors such as cultural norms, caregiver attitudes and beliefs, and contextual factors could be explored to better understand the reasons behind the observed patterns.\n"
      ],
      "metadata": {
        "id": "yq8rtbf0HcIJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['prov_best'])[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children','count_staff_skills',]].mean()"
      ],
      "metadata": {
        "id": "t5hsybyJHjJ2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['prov_best'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female','count_staff_salary_paid','target']].mean()"
      ],
      "metadata": {
        "id": "Ppg67v9MIjAA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[Train.filter(like='id_').columns[0:4]]"
      ],
      "metadata": {
        "id": "GLJXBQxBo3NE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[Train.filter(like='id_').columns[35:40]]"
      ],
      "metadata": {
        "id": "wDt92I0ZWPWv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['id_mn_best','id_dc_best','id_enumerator','id_facility']]"
      ],
      "metadata": {
        "id": "vVQuKKRPMTmj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['id_facility_n','id_ward_n','id_mn_n','id_dc_n','id_prov_n']]"
      ],
      "metadata": {
        "id": "1lNh2Ou1W9m7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id_counts=Train.groupby(['prov_best'])[['id_mn_best','id_dc_best','id_enumerator','id_facility']].transform('count')\n",
        "Train['id_mn_best_count'] = id_counts['id_mn_best']\n",
        "Train['id_dc_best_count'] = id_counts['id_dc_best']\n",
        "Train['id_enumerator_count'] = id_counts['id_enumerator']\n",
        "Train['id_facility_count'] = id_counts['id_facility']"
      ],
      "metadata": {
        "id": "jjH77MFSNN9r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['prov_best'])[['id_facility_n','id_ward_n','id_mn_n','id_dc_n','id_prov_n']].mean()"
      ],
      "metadata": {
        "id": "d8xH9e5QL5OO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##pri"
      ],
      "metadata": {
        "id": "RBum4QPZZNIV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pri_cols = Train.filter(like='pri')\n",
        "missing_cols = plot_missing_cols(pri_cols,threshold=0.6,plot=False)"
      ],
      "metadata": {
        "id": "SRSLziF6ZPHG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pri_cols.drop(missing_cols,axis=1,inplace=True)"
      ],
      "metadata": {
        "id": "-DMBXbKOZPTH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[pri_cols.columns[:10]]"
      ],
      "metadata": {
        "id": "V1WtsgBXZPhR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pri_cols.filter(like='time')"
      ],
      "metadata": {
        "id": "8bJrj7Tf9t87"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['pri_calc_time_close'] = pd.to_datetime(Train['pri_calc_time_close'],format='%H:%M')\n",
        "Train['pri_calc_time_open'] = pd.to_datetime(Train['pri_calc_time_open'],format='%H:%M')\n",
        "Train['pri_calc_diff_hours'] = (Train['pri_calc_time_close'] - Train['pri_calc_time_open']).dt.components['hours']\n",
        "Train['pri_calc_diff_minutes'] = (Train['pri_calc_time_close'] - Train['pri_calc_time_open']).dt.components['minutes']"
      ],
      "metadata": {
        "id": "9YNzEcTm9IoS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sns.catplot(data=Train,x='pri_time_open_hours',y='target',kind='box')"
      ],
      "metadata": {
        "id": "fVVqPoJHB6Xl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sns.catplot(data=Train,x='pri_time_close_hours',y='target',kind='box',height=7)"
      ],
      "metadata": {
        "id": "5aa__TP9CU4N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sns.catplot(data=Train,x='pri_calc_diff_hours',y='target',kind='box',height=7)"
      ],
      "metadata": {
        "id": "dsfkmfqFEh0s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['pri_holidays']].value_counts()"
      ],
      "metadata": {
        "id": "CSU1WC4KwqJf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_holidays'])[['target']].mean()"
      ],
      "metadata": {
        "id": "fgstPzefySYE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_holidays','target_cat'])[['target_cat']].count()"
      ],
      "metadata": {
        "id": "ZTcxPsH1ZP6V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_holidays','prov_best'])[['prov_best']].count()\n",
        "total_counts = Train.groupby(['pri_holidays']).size()\n",
        "grouped['count'] = grouped['prov_best']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='count')"
      ],
      "metadata": {
        "id": "MhldTKZEZQC8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_holidays'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female','count_staff_salary_paid']].mean()"
      ],
      "metadata": {
        "id": "iSMBmSwZZQNi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_holidays'])[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children','count_staff_skills',]].mean()"
      ],
      "metadata": {
        "id": "zWXnFfTrZQWZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_holidays'])[['pri_time_open_hours','pri_time_close_hours','pri_calc_diff_hours']].mean()"
      ],
      "metadata": {
        "id": "NlxptZhaZQfJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['prov_best'])[['pri_time_open_hours','pri_time_close_hours','pri_calc_diff_hours']].mean()"
      ],
      "metadata": {
        "id": "vWmi6KwMFaiA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[pri_cols.columns[10:25]]"
      ],
      "metadata": {
        "id": "a0IOQh6hZQnv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "afrikaans = Train.groupby(['pri_language_1'])[['target']].mean().rename(columns={'target':'Afrikaans_target'})\n",
        "\n",
        "english = Train.groupby(['pri_language_2'])[['target']].mean().rename(columns={'target':'English_target'})\n",
        "\n",
        "isiXhosa = Train.groupby(['pri_language_4'])[['target']].mean().rename(columns={'target':'IsiXhosa_target'})\n",
        "\n",
        "isiZulu = Train.groupby(['pri_language_5'])[['target']].mean().rename(columns={'target':'IsiZulu_target'})\n",
        "\n",
        "sepedi = Train.groupby(['pri_language_6'])[['target']].mean().rename(columns={'target':'Sepedi_target'})\n",
        "\n",
        "setswana = Train.groupby(['pri_language_7'])[['target']].mean().rename(columns={'target':'Setswana_target'})\n",
        "\n",
        "sesotho = Train.groupby(['pri_language_8'])[['target']].mean().rename(columns={'target':'Sesotho_target'})\n",
        "\n",
        "tshivenda = Train.groupby(['pri_language_10'])[['target']].mean().rename(columns={'target':'Tshivenda_target'})\n",
        "\n",
        "xitsonga = Train.groupby(['pri_language_11'])[['target']].mean().rename(columns={'target':'Xitsonga_target'})\n",
        "\n",
        "pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        ""
      ],
      "metadata": {
        "id": "FHA5afyeaHiw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_language_1','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_1']).size()\n",
        "grouped['Afrikaans_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "afrikaans = grouped.filter(like='count')\n",
        "\n",
        "\n",
        "grouped = Train.groupby(['pri_language_2','child_observe_attentive',])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_2']).size()\n",
        "grouped['English_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "english = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_4','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_4']).size()\n",
        "grouped['isiXhosa_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiXhosa = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_5','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_5']).size()\n",
        "grouped['isiZulu_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiZulu = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_6','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_6']).size()\n",
        "grouped['Sepedi_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sepedi = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_7','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_7']).size()\n",
        "grouped['Setswana_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "setswana = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_8','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_8']).size()\n",
        "grouped['Sesotho_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sesotho = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_10','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_10']).size()\n",
        "grouped['Tshivenda_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "tshivenda = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_11','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_language_11']).size()\n",
        "grouped['Xitsonga_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "xitsonga = grouped.filter(like='count')\n",
        "\n",
        "language_attentive = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        "language_attentive = language_attentive.reset_index()\n",
        "\n",
        "no_row1=language_attentive.loc[0] + language_attentive.loc[2]\n",
        "no_row2 = language_attentive.loc[1] + language_attentive.loc[3]\n",
        "yes_row1=language_attentive.loc[4] + language_attentive.loc[6]\n",
        "yes_row2 = language_attentive.loc[5] + language_attentive.loc[7]\n",
        "\n",
        "language_attentive.loc[0] = no_row1\n",
        "language_attentive.loc[1] = no_row2\n",
        "language_attentive.loc[2] = yes_row1\n",
        "language_attentive.loc[3] = yes_row2\n",
        "\n",
        "language_attentive.drop([4,5,6,7],axis=0,inplace=True)\n",
        "language_attentive[language_attentive.filter(like='_count').columns] = language_attentive.filter(like='_count')/2"
      ],
      "metadata": {
        "id": "ZnUhjzV3U4P0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_language_1','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_1']).size()\n",
        "grouped['Afrikaans_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "afrikaans = grouped.filter(like='count')\n",
        "\n",
        "\n",
        "grouped = Train.groupby(['pri_language_2','child_observe_concentrated',])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_2']).size()\n",
        "grouped['English_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "english = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_4','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_4']).size()\n",
        "grouped['isiXhosa_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiXhosa = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_5','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_5']).size()\n",
        "grouped['isiZulu_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiZulu = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_6','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_6']).size()\n",
        "grouped['Sepedi_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sepedi = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_7','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_7']).size()\n",
        "grouped['Setswana_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "setswana = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_8','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_8']).size()\n",
        "grouped['Sesotho_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sesotho = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_10','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_10']).size()\n",
        "grouped['Tshivenda_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "tshivenda = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_11','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_language_11']).size()\n",
        "grouped['Xitsonga_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "xitsonga = grouped.filter(like='count')\n",
        "\n",
        "language_concentrated = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        "language_concentrated = language_concentrated.reset_index()\n",
        "\n",
        "no_row1=language_concentrated.loc[0] + language_concentrated.loc[2]\n",
        "no_row2 = language_concentrated.loc[1] + language_concentrated.loc[3]\n",
        "yes_row1=language_concentrated.loc[4] + language_concentrated.loc[6]\n",
        "yes_row2 = language_concentrated.loc[5] + language_concentrated.loc[7]\n",
        "\n",
        "language_concentrated.loc[0] = no_row1\n",
        "language_concentrated.loc[1] = no_row2\n",
        "language_concentrated.loc[2] = yes_row1\n",
        "language_concentrated.loc[3] = yes_row2\n",
        "\n",
        "language_concentrated.drop([4,5,6,7],axis=0,inplace=True)\n",
        "language_concentrated[language_concentrated.filter(like='_count').columns] = language_concentrated.filter(like='_count')/2"
      ],
      "metadata": {
        "id": "WwybwrCUYA6Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_language_1','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_1']).size()\n",
        "grouped['Afrikaans_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "afrikaans = grouped.filter(like='count')\n",
        "\n",
        "\n",
        "grouped = Train.groupby(['pri_language_2','child_observe_diligent',])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_2']).size()\n",
        "grouped['English_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "english = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_4','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_4']).size()\n",
        "grouped['isiXhosa_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiXhosa = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_5','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_5']).size()\n",
        "grouped['isiZulu_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiZulu = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_6','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_6']).size()\n",
        "grouped['Sepedi_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sepedi = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_7','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_7']).size()\n",
        "grouped['Setswana_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "setswana = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_8','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_8']).size()\n",
        "grouped['Sesotho_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sesotho = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_10','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_10']).size()\n",
        "grouped['Tshivenda_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "tshivenda = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_11','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_language_11']).size()\n",
        "grouped['Xitsonga_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "xitsonga = grouped.filter(like='count')\n",
        "\n",
        "language_diligent = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        "language_diligent = language_diligent.reset_index()\n",
        "\n",
        "no_row1=language_diligent.loc[0] + language_diligent.loc[2]\n",
        "no_row2 = language_diligent.loc[1] + language_diligent.loc[3]\n",
        "yes_row1=language_diligent.loc[4] + language_diligent.loc[6]\n",
        "yes_row2 = language_diligent.loc[5] + language_diligent.loc[7]\n",
        "\n",
        "language_diligent.loc[0] = no_row1\n",
        "language_diligent.loc[1] = no_row2\n",
        "language_diligent.loc[2] = yes_row1\n",
        "language_diligent.loc[3] = yes_row2\n",
        "\n",
        "language_diligent.drop([4,5,6,7],axis=0,inplace=True)\n",
        "language_diligent[language_diligent.filter(like='_count').columns] = language_diligent.filter(like='_count')/2"
      ],
      "metadata": {
        "id": "ZYZMRZ_NegQu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_language_1','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_1']).size()\n",
        "grouped['Afrikaans_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "afrikaans = grouped.filter(like='count')\n",
        "\n",
        "\n",
        "grouped = Train.groupby(['pri_language_2','child_observe_interested',])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_2']).size()\n",
        "grouped['English_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "english = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_4','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_4']).size()\n",
        "grouped['isiXhosa_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiXhosa = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_5','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_5']).size()\n",
        "grouped['isiZulu_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiZulu = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_6','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_6']).size()\n",
        "grouped['Sepedi_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sepedi = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_7','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_7']).size()\n",
        "grouped['Setswana_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "setswana = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_8','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_8']).size()\n",
        "grouped['Sesotho_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sesotho = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_10','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_10']).size()\n",
        "grouped['Tshivenda_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "tshivenda = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_language_11','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_language_11']).size()\n",
        "grouped['Xitsonga_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "xitsonga = grouped.filter(like='count')\n",
        "\n",
        "language_interested = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        "language_interested = language_interested.reset_index()\n",
        "\n",
        "no_row1=language_interested.loc[0] + language_interested.loc[2]\n",
        "no_row2 = language_interested.loc[1] + language_interested.loc[3]\n",
        "yes_row1=language_interested.loc[4] + language_interested.loc[6]\n",
        "yes_row2 = language_interested.loc[5] + language_interested.loc[7]\n",
        "\n",
        "language_interested.loc[0] = no_row1\n",
        "language_interested.loc[1] = no_row2\n",
        "language_interested.loc[2] = yes_row1\n",
        "language_interested.loc[3] = yes_row2\n",
        "\n",
        "language_interested.drop([4,5,6,7],axis=0,inplace=True)\n",
        "language_interested[language_interested.filter(like='_count').columns] = language_interested.filter(like='_count')/2"
      ],
      "metadata": {
        "id": "Df5rVoXYiYHi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "average_language_observe = language_interested.copy()\n",
        "average_language_observe[average_language_observe.filter(like='count').columns] = (language_interested.filter(like='count') +\n",
        "                                                                                   language_attentive.filter(like='count') +\n",
        "                                                                                   language_concentrated.filter(like='count') +\n",
        "                                                                                   language_diligent.filter(like='count'))/5\n",
        "average_language_observe\n",
        "#Positive Observe levels tend to go up when the language for learning and teaching are Afrikaans, English, isiZulu, Sepedi, Tshivenda, Xitsonga\n",
        "#Negative Observe levels tend to go up when the language for learning and teaching are isiXhosa, Setswana, Sesotho"
      ],
      "metadata": {
        "id": "peAcabSPnV0W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['prov_best','pri_language_1'])[['pri_language_1']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Afrikaans_count'] = grouped['pri_language_1']/total_counts[grouped.index.get_level_values(0)].values\n",
        "afrikaans = grouped.filter(like='count')\n",
        "\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_2'])[['pri_language_2']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['English_count'] = grouped['pri_language_2']/total_counts[grouped.index.get_level_values(0)].values\n",
        "english = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_4',])[['pri_language_4']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['isiXhosa_count'] = grouped['pri_language_4']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiXhosa = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_5',])[['pri_language_5']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['isiZulu_count'] = grouped['pri_language_5']/total_counts[grouped.index.get_level_values(0)].values\n",
        "isiZulu = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_6'])[['pri_language_6',]].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Sepedi_count'] = grouped['pri_language_6']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sepedi = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_7'])[['pri_language_7']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Setswana_count'] = grouped['pri_language_7']/total_counts[grouped.index.get_level_values(0)].values\n",
        "setswana = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_8'])[['pri_language_8']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Sesotho_count'] = grouped['pri_language_8']/total_counts[grouped.index.get_level_values(0)].values\n",
        "sesotho = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_10'])[['pri_language_10']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Tshivenda_count'] = grouped['pri_language_10']/total_counts[grouped.index.get_level_values(0)].values\n",
        "tshivenda = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['prov_best','pri_language_11'])[['pri_language_11']].count()\n",
        "total_counts = Train.groupby(['prov_best']).size()\n",
        "grouped['Xitsonga_count'] = grouped['pri_language_11']/total_counts[grouped.index.get_level_values(0)].values\n",
        "xitsonga = grouped.filter(like='count')\n"
      ],
      "metadata": {
        "id": "THWXdVFVuq8G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "language_count = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga],axis=1)\n",
        "language_count"
      ],
      "metadata": {
        "id": "8KuDCCMBvc3S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "afrikaans = Train.groupby(['pri_language_1'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "afrikaans['language'] = 'Afrikaans'\n",
        "\n",
        "english = Train.groupby(['pri_language_2'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "english['language'] = 'English'\n",
        "\n",
        "isiXhosa = Train.groupby(['pri_language_4'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "isiXhosa['language'] = 'isiXhosa'\n",
        "\n",
        "isiZulu = Train.groupby(['pri_language_5'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "isiZulu['language'] = 'isiZulu'\n",
        "\n",
        "sepedi = Train.groupby(['pri_language_6'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "sepedi['language'] = 'Sepedi'\n",
        "\n",
        "setswana = Train.groupby(['pri_language_7'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "setswana['language'] = 'Setswana'\n",
        "\n",
        "sesotho = Train.groupby(['pri_language_8'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "sesotho['language'] = 'Sesotho'\n",
        "\n",
        "tshivenda = Train.groupby(['pri_language_10'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "tshivenda['language'] = 'Tshivenda'\n",
        "\n",
        "xitsonga = Train.groupby(['pri_language_11'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours']].mean()\n",
        "xitsonga['language'] = 'Xitsonga'"
      ],
      "metadata": {
        "id": "0CaJH5MPK8V9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "language = pd.concat([afrikaans,english,isiXhosa,isiZulu,sepedi,setswana,sesotho,tshivenda,xitsonga])\n",
        "language = language.reset_index()\n",
        "language.set_index(['language','index'])"
      ],
      "metadata": {
        "id": "YeQmp-PJNrI4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['pri_separate'].value_counts()"
      ],
      "metadata": {
        "id": "OZnAISiMZQv8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_separate','target_cat'])[['target_cat']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['count'] = grouped['target_cat']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='count')"
      ],
      "metadata": {
        "id": "gQSbflbQZQ5i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_separate','child_observe_attentive'])[['child_observe_attentive']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['child_observe_attentive_count'] = grouped['child_observe_attentive']/total_counts[grouped.index.get_level_values(0)].values\n",
        "att = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_separate','child_observe_concentrated'])[['child_observe_concentrated']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['child_observe_concentrated_count'] = grouped['child_observe_concentrated']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ac = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_separate','child_observe_interested'])[['child_observe_interested']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['child_observe_interested_count'] = grouped['child_observe_interested']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ai = grouped.filter(like='count')\n",
        "\n",
        "grouped = Train.groupby(['pri_separate','child_observe_diligent'])[['child_observe_diligent']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['child_observe_diligent_count'] = grouped['child_observe_diligent']/total_counts[grouped.index.get_level_values(0)].values\n",
        "ad = grouped.filter(like='count')\n",
        "\n",
        "att = pd.concat([att,ac['child_observe_concentrated_count'],ai['child_observe_interested_count'],ad['child_observe_diligent_count']],axis=1)"
      ],
      "metadata": {
        "id": "3kKGl4sAZRBT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "att.filter(like='count').mean(axis=1)"
      ],
      "metadata": {
        "id": "QuFzGOMTZRLr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grouped = Train.groupby(['pri_separate','prov_best'])[['prov_best']].count()\n",
        "total_counts = Train.groupby(['pri_separate']).size()\n",
        "grouped['count'] = grouped['prov_best']/total_counts[grouped.index.get_level_values(0)].values\n",
        "grouped.filter(like='count')"
      ],
      "metadata": {
        "id": "4gSOwmt2ZRU0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_separate'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female','count_staff_salary_paid']].mean()"
      ],
      "metadata": {
        "id": "Zt7lkmhhQke0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_separate'])[['count_children_present','count_children_attendance','count_children_precovid','count_toilets_children','count_staff_skills',]].mean()"
      ],
      "metadata": {
        "id": "JB238FSbTzcr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_separate'])[['pri_time_open_hours','pri_time_close_hours','pri_calc_diff_hours']].mean()"
      ],
      "metadata": {
        "id": "0uAX-6hyTzqO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[pri_cols.columns[25:35]]"
      ],
      "metadata": {
        "id": "YRzWIZRlcvti"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_transport'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "GKyTqlsH99hE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_meal'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "yUqavC8p6Bvj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_bank'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "u3EObMA_4YhG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_land'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "fJ5kIGO3qE4t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_facilities'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "E_PEbIPkZwZh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['pri_fees','pri_fees_amount','pri_fees_free']]"
      ],
      "metadata": {
        "id": "CviZjrxXUf-w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(Train['pri_fees'].value_counts()/len(Train['pri_fees']))\n",
        "print(Train['pri_fees_free'].value_counts()/len(Train['pri_fees_free']))"
      ],
      "metadata": {
        "id": "r5QbEGImU8gY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_fees'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "hHEIpVgcWrIM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_fees_free'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "bb5bgJgDXYM9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['pri_aftercare'].value_counts()"
      ],
      "metadata": {
        "id": "9ZQOM7wicv3i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_aftercare'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "sgGs7GAYUWzj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vard[pri_cols.columns[35:45]]"
      ],
      "metadata": {
        "id": "FXVMuhRb-GX4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train[['pri_meal_1','pri_meal_3','pri_meal_4','pri_meal_2','pri_meals']]"
      ],
      "metadata": {
        "id": "QsSFM4Wf-Ijk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train['pri_meals'].value_counts()"
      ],
      "metadata": {
        "id": "McLqnNcMBuLK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Train.groupby(['pri_meals'])[['count_register_race_african','count_register_race_coloured','count_staff_gender_female',\n",
        "                                   'count_staff_salary_paid','count_children_present','count_children_attendance','count_children_precovid',\n",
        "                                   'count_toilets_children','count_staff_skills','pri_time_open_hours','pri_time_close_hours','pri_fees_amount']].mean()"
      ],
      "metadata": {
        "id": "3qA3zyOPBuSm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JTS3TQ0WBudL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qTxTQIoGzQU0"
      },
      "source": [
        "#Feature Engineering"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u000yCjzBZwa"
      },
      "outputs": [],
      "source": [
        "def logic_mapping(att, conc, dilig, interest):\n",
        "  logic_map = {'Sometimes':False,'Often':True,'Almost always':True,'Almost never':False}\n",
        "  result_and = (logic_map[att]) and (logic_map[conc]) and (logic_map[dilig]) and (logic_map[interest])\n",
        "  result_or = (logic_map[att]) or (logic_map[conc]) or (logic_map[dilig]) or (logic_map[interest])\n",
        "  result_nand = (not(logic_map[att])) and (not(logic_map[conc])) and (not(logic_map[dilig])) and (not(logic_map[interest]))\n",
        "  result_nor = (not(logic_map[att])) and (not(logic_map[conc])) and (not(logic_map[dilig])) and (not(logic_map[interest]))\n",
        "  logic_remap = {True:0,False:1}\n",
        "  return logic_remap[result_and], logic_remap[result_or], logic_remap[result_nand], logic_remap[result_nor]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aD-SEOi8jvOq"
      },
      "outputs": [],
      "source": [
        "attention_mapping = {'Sometimes': 2, 'Often': 3, 'Almost always': 4, 'Almost never': 1}\n",
        "programme_year_mapping = {'1st year in the programme':1, '2nd year in programme': 2, '3rd year in programme':3, 'Do Not Know':4}\n",
        "Logic_ops = data[['child_observe_attentive','child_observe_concentrated','child_observe_diligent','child_observe_interested']].apply(\n",
        "    lambda row: logic_mapping(row[0],row[1],row[2],row[3]), axis=1)\n",
        "data['AND_Observe'] = Logic_ops.apply(lambda row: row[0])\n",
        "data['OR_Observe'] = Logic_ops.apply(lambda row: row[1])\n",
        "data['NAND_Observe'] = Logic_ops.apply(lambda row: row[2])\n",
        "data['NOR_Observe'] = Logic_ops.apply(lambda row: row[3])\n",
        "data['child_years_in_programme'] = data['child_years_in_programme'].map(programme_year_mapping)\n",
        "data['age_bin'] = pd.qcut(data['child_age'],5,labels=False)\n",
        "data['child_observe_attentive'] = data['child_observe_attentive'].map(attention_mapping)\n",
        "data['child_observe_concentrated'] = data['child_observe_concentrated'].map(attention_mapping)\n",
        "data['child_observe_diligent'] = data['child_observe_diligent'].map(attention_mapping)\n",
        "data['child_observe_interested'] = data['child_observe_interested'].map(attention_mapping)\n",
        "data['child_observe_totals'] = data['child_observe_diligent'] * 4 + data['child_observe_attentive'] * 3 + data['child_observe_concentrated'] * 2 + data['child_observe_interested'] * 1\n",
        "data['child_observe_totals*'] = (data['child_observe_diligent'] * 4) * (data['child_observe_attentive'] * 3) * (data['child_observe_concentrated'] * 2) * (data['child_observe_interested'] * 1)\n",
        "data['weighted_child_age+attention'] = data['child_observe_diligent'] + data['age_bin'] + 1 + data['child_observe_attentive']\n",
        "data['weighted_child_age*attention'] = data['child_observe_diligent'] * (data['age_bin']+1) * data['child_observe_attentive']\n",
        "data['weighted_child_age+prog_year+attention'] = data['child_years_in_programme'] + data['child_observe_diligent'] + data['age_bin'] + 1 + data['child_observe_attentive']\n",
        "data['weighted_child_age*prog_year*attention'] = data['child_years_in_programme'] * data['child_observe_diligent'] * (data['age_bin']+1) * data['child_observe_attentive']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yVsgnbCLUlSA"
      },
      "outputs": [],
      "source": [
        "data['Year_dob'] = data['child_dob'].str.split('-',expand=True)[0].astype(float)\n",
        "data['Month_dob'] = data['child_dob'].str.split('-',expand=True)[1].astype(float)\n",
        "data['Day_dob'] = data['child_dob'].str.split('-',expand=True)[2].astype(float)\n",
        "data['child_gender'] = data['child_gender'].map({'Female':1,'Male':2})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ilx88cXYjwOE"
      },
      "outputs": [],
      "source": [
        "data['Count_observe_totals_level_per_age_group'] = data.groupby(['age_bin','child_observe_totals'])[['child_observe_totals']].transform('count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7hitfskzjwTd"
      },
      "outputs": [],
      "source": [
        "data['Mean_observe_totals_level_per_age_group'] = data.groupby(['age_bin','child_observe_totals'])[['child_observe_totals']].transform('mean')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FtN9oBGqjwdy"
      },
      "outputs": [],
      "source": [
        "data['sum_observe_totals_level_per_age_group'] = data.groupby(['age_bin','child_observe_totals'])[['child_observe_totals']].transform('sum')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "68xLtvKWjwjA"
      },
      "outputs": [],
      "source": [
        "data['Mean_weighted_attentive_level_per_gender'] = data.groupby(['child_gender'])[['weighted_child_age+prog_year+attention']].transform('mean')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4EKG1Wwijwox"
      },
      "outputs": [],
      "source": [
        "data['Mean_weighted_attentive*_level_per_gender'] = data.groupby(['child_gender'])[['weighted_child_age*prog_year*attention']].transform('mean')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vJ6u9yenqRfh"
      },
      "outputs": [],
      "source": [
        "new_Train = data[data['Train']=='Train']\n",
        "new_Test = data[data['Train']=='Test']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GwKXtVGjJUJ7"
      },
      "source": [
        "#Baseline Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MedYJsfXLc25"
      },
      "outputs": [],
      "source": [
        "def Baseline_Pred(model, X, y, test, scaler=None, verbose=False, predictions=True):\n",
        "  fold = KFold(n_splits=5)\n",
        "  test_preds = []\n",
        "  oof_f1 = []\n",
        "  for train_index, test_index in fold.split(X, y):\n",
        "    X_train, X_test = X.iloc[train_index],X.iloc[test_index]\n",
        "    y_train, y_test = y.iloc[train_index],y.iloc[test_index]\n",
        "    model.fit(X_train, y_train, eval_set=(X_test, y_test), verbose=verbose)\n",
        "    preds=model.predict(X_test)\n",
        "    if predictions:\n",
        "      test_pred = model.predict(test[X.columns])\n",
        "      test_preds.append(test_pred)\n",
        "    print(f\"RMSE {mean_squared_error(y_test,preds,squared=False)}\")\n",
        "    oof_f1.append(mean_squared_error(y_test,preds,squared=False))\n",
        "\n",
        "  print(f\"Average_RMSE {sum(oof_f1)/5}\")\n",
        "  if predictions:\n",
        "    return test_preds, model\n",
        "  return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sBWtLTPl7nuY"
      },
      "source": [
        "###Numerical Columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gEBos6vw7m5x"
      },
      "outputs": [],
      "source": [
        "# num_cols = list(set(Test.select_dtypes('number')).intersection(Train.select_dtypes('number')))\n",
        "num_cols = new_Train.select_dtypes('number').columns\n",
        "num_train = new_Train[num_cols]\n",
        "#num_test = Test[num_cols]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eKt2h3y898p2"
      },
      "outputs": [],
      "source": [
        "missing_train_cols= plot_missing_cols(num_train, threshold=0.8, plot=False)\n",
        "#missing_test_cols = plot_missing_cols(num_test, threshold=0.8, plot=False)\n",
        "num_Train = num_train.drop(missing_train_cols,axis=1)\n",
        "#num_Test = num_test.drop(missing_test_cols,axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EuuNhnMG9FGH"
      },
      "outputs": [],
      "source": [
        "model_cbc = CatBoostRegressor(eval_metric='RMSE')\n",
        "model_lgb = LGBMRegressor()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IdwPQDb776Kx"
      },
      "outputs": [],
      "source": [
        "num_Train.drop(['target','child_age','child_height','child_observe_total'],axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YTFwUYpTrTQC"
      },
      "outputs": [],
      "source": [
        "cbc_model = Baseline_Pred(model_cbc, num_Train, Train['target'], Test, verbose=False, predictions=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4jSM6Nzls2gW"
      },
      "outputs": [],
      "source": [
        "features_cbc = pd.DataFrame({'feature': list(num_Train.columns), 'importance': cbc_model.feature_importances_}).sort_values('importance', ascending = False)\n",
        "features_cbc.head(30)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bllKBZbD76xF"
      },
      "source": [
        "###Categorical Columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L_APzE_ZITD2"
      },
      "outputs": [],
      "source": [
        "\n",
        "cat_cols = Train.select_dtypes(include=['object']).columns\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2MgcCySFwMnD"
      },
      "outputs": [],
      "source": [
        "le = LabelEncoder()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eenr5AUrTmyp"
      },
      "outputs": [],
      "source": [
        "test_cat_cols = Test.select_dtypes('object').columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e4cWi6Z5USbj"
      },
      "outputs": [],
      "source": [
        "cat_train = Train[cat_cols]\n",
        "cat_test = Test[test_cat_cols]\n",
        "missing_cols_train = plot_missing_cols(cat_train,threshold=0.8,plot=False)\n",
        "missing_cols_test = plot_missing_cols(cat_test, threshold=0.8,plot=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oNh3M7nCWZtX"
      },
      "outputs": [],
      "source": [
        "cat_Train = cat_train.drop(missing_cols_train,axis=1)\n",
        "cat_Test = cat_test.drop(missing_cols_test,axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5p59k_DvXwqa"
      },
      "outputs": [],
      "source": [
        "for col in cat_Train.columns:\n",
        "  cat_Train[col] = le.fit_transform(cat_Train[col])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_9aob_35X8og"
      },
      "outputs": [],
      "source": [
        "for col in cat_Test.columns:\n",
        "  cat_Test[col] = le.fit_transform(cat_Test[col])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IEDEjTSTYGHt"
      },
      "outputs": [],
      "source": [
        "cat_Train.drop(['child_id'],axis=1,inplace=True)\n",
        "cat_Test.drop(['child_id'],axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q2_yP_qOcXft"
      },
      "outputs": [],
      "source": [
        "model_cbc = CatBoostRegressor(eval_metric='RMSE')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "umv55D1PSrue"
      },
      "outputs": [],
      "source": [
        "cbc_preds, cbc_model = Baseline_Pred(model_cbc, cat_Train, Train['target'], cat_Test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vDPv-3ZP9d-I"
      },
      "source": [
        "##Submission"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qB9xDBMNemDw"
      },
      "outputs": [],
      "source": [
        "explainer = shap.Explainer(cbc_model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y0xqSSl9cQXY"
      },
      "outputs": [],
      "source": [
        "# Compute the SHAP values for your test dataset\n",
        "shap_values = explainer(num_Test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dt5x92wlbOjX"
      },
      "outputs": [],
      "source": [
        "xd = num_Test.copy()\n",
        "xd[num_Test.columns] = shap_values.values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GTtA7FFRlucM"
      },
      "outputs": [],
      "source": [
        "top_cols = xd.apply(lambda x: x.nlargest(15).index.tolist(), axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOtCkvHnkxAt"
      },
      "outputs": [],
      "source": [
        "feature_cols = Sub.filter(like='feature').columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cj-8Seg_qHlc"
      },
      "outputs": [],
      "source": [
        "xdc = pd.DataFrame(columns=feature_cols)\n",
        "for i, cols in enumerate(top_cols):\n",
        "  xdc.loc[i] = cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lq_WEvsIrgeh"
      },
      "outputs": [],
      "source": [
        "Sub['target'] = sum(cbc_preds)/5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HNIPLuKAs0ba"
      },
      "outputs": [],
      "source": [
        "Sub[feature_cols] = xdc[feature_cols]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lRTBLAvftCCE"
      },
      "outputs": [],
      "source": [
        "Sub.to_csv('/content/Baseline_Only_Numerical_Columns_above_0.8threshold_CatBoost_Sub.csv',index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wGz3whuQaqns"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "bug5r_Hkz7no",
        "XHwCHPsQEL0_",
        "MPtyzsTR0C_5",
        "2aALNzhtl8Pt",
        "DpaWYXDHkgkX",
        "XwW4G0h3D158",
        "4TivrDKi2p7m",
        "vZ-AFymp2y8P",
        "isAsIwQrKVqF",
        "cn5Za1u2D3qP",
        "bMB1uMZ1D3z-",
        "bllKBZbD76xF",
        "vDPv-3ZP9d-I"
      ],
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}