1_historical_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dotenv\n",
    "\n",
    "dotenv.load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import kagglehub\n",
    "\n",
    "# Download latest version\n",
    "path = kagglehub.dataset_download(\"alanvourch/tmdb-movies-daily-updates\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in dataset directory: ['TMDB_all_movies.csv']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>status</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>budget</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>...</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>director_of_photography</th>\n",
       "      <th>writers</th>\n",
       "      <th>producers</th>\n",
       "      <th>music_composer</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>imdb_votes</th>\n",
       "      <th>poster_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>Ariel</td>\n",
       "      <td>7.1</td>\n",
       "      <td>335.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1988-10-21</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>tt0094675</td>\n",
       "      <td>...</td>\n",
       "      <td>suomi</td>\n",
       "      <td>Heikki Salomaa, Hanna Jokinen, Matti Pellonpää...</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.4</td>\n",
       "      <td>8870.0</td>\n",
       "      <td>/ojDg0PGvs6R9xYFodRct2kdI6wC.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Shadows in Paradise</td>\n",
       "      <td>7.3</td>\n",
       "      <td>400.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1986-10-17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>tt0092149</td>\n",
       "      <td>...</td>\n",
       "      <td>English, suomi, svenska</td>\n",
       "      <td>Teuvo Rissanen, Malla Hukkanen, Olli Varja, Ju...</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Mika Kaurismäki</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.5</td>\n",
       "      <td>7654.0</td>\n",
       "      <td>/nj01hspawPof0mJmlgfjuLyJuRN.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Four Rooms</td>\n",
       "      <td>5.9</td>\n",
       "      <td>2650.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1995-12-09</td>\n",
       "      <td>4257354.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>4000000.0</td>\n",
       "      <td>tt0113101</td>\n",
       "      <td>...</td>\n",
       "      <td>English</td>\n",
       "      <td>Lili Taylor, Valeria Golino, Marisa Tomei, Mar...</td>\n",
       "      <td>Quentin Tarantino, Robert Rodriguez, Allison A...</td>\n",
       "      <td>Andrzej Sekula, Rodrigo García, Guillermo Nava...</td>\n",
       "      <td>Quentin Tarantino, Robert Rodriguez, Allison A...</td>\n",
       "      <td>Quentin Tarantino, Lawrence Bender, Alexandre ...</td>\n",
       "      <td>Combustible Edison</td>\n",
       "      <td>6.7</td>\n",
       "      <td>113089.0</td>\n",
       "      <td>/pyCk5JgtRZwRxnXwfrvyzukaKue.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Judgment Night</td>\n",
       "      <td>6.5</td>\n",
       "      <td>333.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1993-10-15</td>\n",
       "      <td>12136938.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>21000000.0</td>\n",
       "      <td>tt0107286</td>\n",
       "      <td>...</td>\n",
       "      <td>English</td>\n",
       "      <td>Stephen Dorff, Everlast, Will Zahrn, Emilio Es...</td>\n",
       "      <td>Stephen Hopkins</td>\n",
       "      <td>Peter Levy</td>\n",
       "      <td>Jere Cunningham, Lewis Colick</td>\n",
       "      <td>Gene Levy, Marilyn Vance, Lloyd Segan</td>\n",
       "      <td>Alan Silvestri</td>\n",
       "      <td>6.6</td>\n",
       "      <td>19456.0</td>\n",
       "      <td>/3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>Life in Loops (A Megacities RMX)</td>\n",
       "      <td>7.5</td>\n",
       "      <td>27.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>2006-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>tt0825671</td>\n",
       "      <td>...</td>\n",
       "      <td>English, हिन्दी, 日本語, Pусский, Español</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Timo Novotny</td>\n",
       "      <td>Wolfgang Thaler</td>\n",
       "      <td>Michael Glawogger, Timo Novotny</td>\n",
       "      <td>Ulrich Gehmacher, Timo Novotny</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.2</td>\n",
       "      <td>284.0</td>\n",
       "      <td>/7ln81BRnPR2wqxuITZxEciCe1lc.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                             title  vote_average  vote_count    status  \\\n",
       "0   2                             Ariel           7.1       335.0  Released   \n",
       "1   3               Shadows in Paradise           7.3       400.0  Released   \n",
       "2   5                        Four Rooms           5.9      2650.0  Released   \n",
       "3   6                    Judgment Night           6.5       333.0  Released   \n",
       "4   8  Life in Loops (A Megacities RMX)           7.5        27.0  Released   \n",
       "\n",
       "  release_date     revenue  runtime      budget    imdb_id  ...  \\\n",
       "0   1988-10-21         0.0     73.0         0.0  tt0094675  ...   \n",
       "1   1986-10-17         0.0     74.0         0.0  tt0092149  ...   \n",
       "2   1995-12-09   4257354.0     98.0   4000000.0  tt0113101  ...   \n",
       "3   1993-10-15  12136938.0    109.0  21000000.0  tt0107286  ...   \n",
       "4   2006-01-01         0.0     80.0     42000.0  tt0825671  ...   \n",
       "\n",
       "                         spoken_languages  \\\n",
       "0                                   suomi   \n",
       "1                 English, suomi, svenska   \n",
       "2                                 English   \n",
       "3                                 English   \n",
       "4  English, हिन्दी, 日本語, Pусский, Español   \n",
       "\n",
       "                                                cast  \\\n",
       "0  Heikki Salomaa, Hanna Jokinen, Matti Pellonpää...   \n",
       "1  Teuvo Rissanen, Malla Hukkanen, Olli Varja, Ju...   \n",
       "2  Lili Taylor, Valeria Golino, Marisa Tomei, Mar...   \n",
       "3  Stephen Dorff, Everlast, Will Zahrn, Emilio Es...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                            director  \\\n",
       "0                                     Aki Kaurismäki   \n",
       "1                                     Aki Kaurismäki   \n",
       "2  Quentin Tarantino, Robert Rodriguez, Allison A...   \n",
       "3                                    Stephen Hopkins   \n",
       "4                                       Timo Novotny   \n",
       "\n",
       "                             director_of_photography  \\\n",
       "0                                      Timo Salminen   \n",
       "1                                      Timo Salminen   \n",
       "2  Andrzej Sekula, Rodrigo García, Guillermo Nava...   \n",
       "3                                         Peter Levy   \n",
       "4                                    Wolfgang Thaler   \n",
       "\n",
       "                                             writers  \\\n",
       "0                                     Aki Kaurismäki   \n",
       "1                                     Aki Kaurismäki   \n",
       "2  Quentin Tarantino, Robert Rodriguez, Allison A...   \n",
       "3                      Jere Cunningham, Lewis Colick   \n",
       "4                    Michael Glawogger, Timo Novotny   \n",
       "\n",
       "                                           producers      music_composer  \\\n",
       "0                                     Aki Kaurismäki                 NaN   \n",
       "1                                    Mika Kaurismäki                 NaN   \n",
       "2  Quentin Tarantino, Lawrence Bender, Alexandre ...  Combustible Edison   \n",
       "3              Gene Levy, Marilyn Vance, Lloyd Segan      Alan Silvestri   \n",
       "4                     Ulrich Gehmacher, Timo Novotny                 NaN   \n",
       "\n",
       "  imdb_rating imdb_votes                       poster_path  \n",
       "0         7.4     8870.0  /ojDg0PGvs6R9xYFodRct2kdI6wC.jpg  \n",
       "1         7.5     7654.0  /nj01hspawPof0mJmlgfjuLyJuRN.jpg  \n",
       "2         6.7   113089.0  /pyCk5JgtRZwRxnXwfrvyzukaKue.jpg  \n",
       "3         6.6    19456.0  /3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg  \n",
       "4         8.2      284.0  /7ln81BRnPR2wqxuITZxEciCe1lc.jpg  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# List files in the downloaded dataset directory\n",
    "files = os.listdir(path)\n",
    "print(\"Files in dataset directory:\", files)\n",
    "\n",
    "# Read the first file\n",
    "df = pd.read_csv(os.path.join(path, files[0]))\n",
    "\n",
    "# Print the first 5 rows\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import preprocess, new_features\n",
    "\n",
    "df = preprocess(df)\n",
    "df = new_features(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>status</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>budget</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>...</th>\n",
       "      <th>producers</th>\n",
       "      <th>music_composer</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>imdb_votes</th>\n",
       "      <th>poster_path</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>first_producer</th>\n",
       "      <th>first_actor</th>\n",
       "      <th>first_company</th>\n",
       "      <th>release_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Four Rooms</td>\n",
       "      <td>5.9</td>\n",
       "      <td>2650.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1995-12-09</td>\n",
       "      <td>4257354.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>4000000.0</td>\n",
       "      <td>tt0113101</td>\n",
       "      <td>...</td>\n",
       "      <td>Quentin Tarantino, Lawrence Bender, Alexandre ...</td>\n",
       "      <td>Combustible Edison</td>\n",
       "      <td>6.7</td>\n",
       "      <td>113089.0</td>\n",
       "      <td>/pyCk5JgtRZwRxnXwfrvyzukaKue.jpg</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>10520</td>\n",
       "      <td>12357</td>\n",
       "      <td>30</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Judgment Night</td>\n",
       "      <td>6.5</td>\n",
       "      <td>333.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1993-10-15</td>\n",
       "      <td>12136938.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>21000000.0</td>\n",
       "      <td>tt0107286</td>\n",
       "      <td>...</td>\n",
       "      <td>Gene Levy, Marilyn Vance, Lloyd Segan</td>\n",
       "      <td>Alan Silvestri</td>\n",
       "      <td>6.6</td>\n",
       "      <td>19456.0</td>\n",
       "      <td>/3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>4249</td>\n",
       "      <td>19212</td>\n",
       "      <td>36</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11</td>\n",
       "      <td>Star Wars</td>\n",
       "      <td>8.2</td>\n",
       "      <td>20722.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1977-05-25</td>\n",
       "      <td>775398007.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>11000000.0</td>\n",
       "      <td>tt0076759</td>\n",
       "      <td>...</td>\n",
       "      <td>Rick McCallum, Gary Kurtz, George Lucas</td>\n",
       "      <td>John Williams</td>\n",
       "      <td>8.6</td>\n",
       "      <td>1488128.0</td>\n",
       "      <td>/6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>10962</td>\n",
       "      <td>11912</td>\n",
       "      <td>36</td>\n",
       "      <td>1977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12</td>\n",
       "      <td>Finding Nemo</td>\n",
       "      <td>7.8</td>\n",
       "      <td>19330.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>2003-05-30</td>\n",
       "      <td>940335536.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>94000000.0</td>\n",
       "      <td>tt0266543</td>\n",
       "      <td>...</td>\n",
       "      <td>John Lasseter, Graham Walters</td>\n",
       "      <td>Thomas Newman</td>\n",
       "      <td>8.2</td>\n",
       "      <td>1143550.0</td>\n",
       "      <td>/eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>6357</td>\n",
       "      <td>1796</td>\n",
       "      <td>36</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>Forrest Gump</td>\n",
       "      <td>8.5</td>\n",
       "      <td>27647.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1994-06-23</td>\n",
       "      <td>677387716.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>55000000.0</td>\n",
       "      <td>tt0109830</td>\n",
       "      <td>...</td>\n",
       "      <td>Wendy Finerman, Steve Tisch, Steve Starkey</td>\n",
       "      <td>Alan Silvestri</td>\n",
       "      <td>8.8</td>\n",
       "      <td>2338068.0</td>\n",
       "      <td>/arw2vcBveWOVZr6pxd9XTd1TdQa.jpg</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>13517</td>\n",
       "      <td>9552</td>\n",
       "      <td>38</td>\n",
       "      <td>1994</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id           title  vote_average  vote_count    status release_date  \\\n",
       "2   5      Four Rooms           5.9      2650.0  Released   1995-12-09   \n",
       "3   6  Judgment Night           6.5       333.0  Released   1993-10-15   \n",
       "6  11       Star Wars           8.2     20722.0  Released   1977-05-25   \n",
       "7  12    Finding Nemo           7.8     19330.0  Released   2003-05-30   \n",
       "8  13    Forrest Gump           8.5     27647.0  Released   1994-06-23   \n",
       "\n",
       "       revenue  runtime      budget    imdb_id  ...  \\\n",
       "2    4257354.0     98.0   4000000.0  tt0113101  ...   \n",
       "3   12136938.0    109.0  21000000.0  tt0107286  ...   \n",
       "6  775398007.0    121.0  11000000.0  tt0076759  ...   \n",
       "7  940335536.0    100.0  94000000.0  tt0266543  ...   \n",
       "8  677387716.0    142.0  55000000.0  tt0109830  ...   \n",
       "\n",
       "                                           producers      music_composer  \\\n",
       "2  Quentin Tarantino, Lawrence Bender, Alexandre ...  Combustible Edison   \n",
       "3              Gene Levy, Marilyn Vance, Lloyd Segan      Alan Silvestri   \n",
       "6            Rick McCallum, Gary Kurtz, George Lucas       John Williams   \n",
       "7                      John Lasseter, Graham Walters       Thomas Newman   \n",
       "8         Wendy Finerman, Steve Tisch, Steve Starkey      Alan Silvestri   \n",
       "\n",
       "  imdb_rating  imdb_votes                       poster_path  timestamp  \\\n",
       "2         6.7    113089.0  /pyCk5JgtRZwRxnXwfrvyzukaKue.jpg 2025-01-08   \n",
       "3         6.6     19456.0  /3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg 2025-01-08   \n",
       "6         8.6   1488128.0  /6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg 2025-01-08   \n",
       "7         8.2   1143550.0  /eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg 2025-01-08   \n",
       "8         8.8   2338068.0  /arw2vcBveWOVZr6pxd9XTd1TdQa.jpg 2025-01-08   \n",
       "\n",
       "  first_producer first_actor first_company release_year  \n",
       "2          10520       12357            30         1995  \n",
       "3           4249       19212            36         1993  \n",
       "6          10962       11912            36         1977  \n",
       "7           6357        1796            36         2003  \n",
       "8          13517        9552            38         1994  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['production_countries'] = pd.Categorical(df['production_countries'])\n",
    "\n",
    "# Changing the release_date to release_year\n",
    "df['release_year'] = pd.to_datetime(df['release_date']).dt.year\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'title', 'vote_average', 'vote_count', 'status', 'release_date',\n",
      "       'revenue', 'runtime', 'budget', 'imdb_id', 'original_language',\n",
      "       'original_title', 'overview', 'popularity', 'tagline', 'genres',\n",
      "       'production_companies', 'production_countries', 'spoken_languages',\n",
      "       'cast', 'director', 'director_of_photography', 'writers', 'producers',\n",
      "       'music_composer', 'imdb_rating', 'imdb_votes', 'poster_path',\n",
      "       'timestamp', 'first_producer', 'first_actor', 'first_company',\n",
      "       'release_year'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-01-08 12:45:05,678 INFO: Initializing external client\n",
      "2025-01-08 12:45:05,679 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
      "2025-01-08 12:45:06,963 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1169559\n",
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1169559/fs/1160262/fg/1394776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 25722/25722 | Elapsed Time: 00:02 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: tmdb_movies_2_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1169559/jobs/named/tmdb_movies_2_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('tmdb_movies_2_offline_fg_materialization', 'SPARK'), None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import hopsworks\n",
    "\n",
    "project = hopsworks.login(project=\"dbillLab1\")\n",
    "fs = project.get_feature_store()\n",
    "\n",
    "tmdb_movie_fg = fs.get_or_create_feature_group(\n",
    "    name='tmdb_movies',\n",
    "    description='Movie information including, director, actor, genres, runtime, averagerating etc',\n",
    "    version=2,\n",
    "    primary_key=['id'],\n",
    "    online_enabled=True,\n",
    ")\n",
    "tmdb_movie_fg.insert(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 25722/25722 | Elapsed Time: 00:01 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: tmdb_movie_genres_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1169559/jobs/named/tmdb_movie_genres_1_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('tmdb_movie_genres_1_offline_fg_materialization', 'SPARK'), None)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import split_genres\n",
    "\n",
    "tmdb_movie_genres_fg = fs.get_or_create_feature_group(\n",
    "    name='tmdb_movie_genres',\n",
    "    description='Movie genres',\n",
    "    version=1,\n",
    "    primary_key=['id'],\n",
    "    online_enabled=True,\n",
    ")\n",
    "\n",
    "df_genres = split_genres(df)\n",
    "tmdb_movie_genres_fg.insert(df_genres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>action</th>\n",
       "      <th>adventure</th>\n",
       "      <th>animation</th>\n",
       "      <th>comedy</th>\n",
       "      <th>crime</th>\n",
       "      <th>documentary</th>\n",
       "      <th>drama</th>\n",
       "      <th>family</th>\n",
       "      <th>...</th>\n",
       "      <th>history</th>\n",
       "      <th>horror</th>\n",
       "      <th>music</th>\n",
       "      <th>mystery</th>\n",
       "      <th>romance</th>\n",
       "      <th>science_fiction</th>\n",
       "      <th>tv_movie</th>\n",
       "      <th>thriller</th>\n",
       "      <th>war</th>\n",
       "      <th>western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  timestamp  action  adventure  animation  comedy  crime  documentary  \\\n",
       "2   5 2025-01-08       0          0          0       1      0            0   \n",
       "3   6 2025-01-08       1          0          0       0      1            0   \n",
       "6  11 2025-01-08       1          1          0       0      0            0   \n",
       "7  12 2025-01-08       0          0          1       0      0            0   \n",
       "8  13 2025-01-08       0          0          0       1      0            0   \n",
       "\n",
       "   drama  family  ...  history  horror  music  mystery  romance  \\\n",
       "2      0       0  ...        0       0      0        0        0   \n",
       "3      0       0  ...        0       0      0        0        0   \n",
       "6      0       0  ...        0       0      0        0        0   \n",
       "7      0       1  ...        0       0      0        0        0   \n",
       "8      1       0  ...        0       0      0        0        1   \n",
       "\n",
       "   science_fiction  tv_movie  thriller  war  western  \n",
       "2                0         0         0    0        0  \n",
       "3                0         0         1    0        0  \n",
       "6                1         0         0    0        0  \n",
       "7                0         0         0    0        0  \n",
       "8                0         0         0    0        0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_genres.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "imdb",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}