2_feature_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dotenv\n",
    "\n",
    "dotenv.load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/martinbravodiaz/miniconda3/envs/imdb/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime, timedelta, timezone\n",
    "import pandas as pd\n",
    "from xgboost import XGBRegressor\n",
    "import hopsworks\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-01-08 04:14:34,063 INFO: Initializing external client\n",
      "2025-01-08 04:14:34,064 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
      "2025-01-08 04:14:35,646 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1169559\n"
     ]
    }
   ],
   "source": [
    "import hopsworks\n",
    "\n",
    "project = hopsworks.login(project=\"dbillLab1\")\n",
    "fs = project.get_feature_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in dataset directory: ['TMDB_all_movies.csv']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>status</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>budget</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>...</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>director_of_photography</th>\n",
       "      <th>writers</th>\n",
       "      <th>producers</th>\n",
       "      <th>music_composer</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>imdb_votes</th>\n",
       "      <th>poster_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>Ariel</td>\n",
       "      <td>7.1</td>\n",
       "      <td>335.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1988-10-21</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>tt0094675</td>\n",
       "      <td>...</td>\n",
       "      <td>suomi</td>\n",
       "      <td>Heikki Salomaa, Hanna Jokinen, Matti Pellonpää...</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.4</td>\n",
       "      <td>8870.0</td>\n",
       "      <td>/ojDg0PGvs6R9xYFodRct2kdI6wC.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Shadows in Paradise</td>\n",
       "      <td>7.3</td>\n",
       "      <td>400.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1986-10-17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>tt0092149</td>\n",
       "      <td>...</td>\n",
       "      <td>English, suomi, svenska</td>\n",
       "      <td>Teuvo Rissanen, Malla Hukkanen, Olli Varja, Ju...</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>Mika Kaurismäki</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.5</td>\n",
       "      <td>7654.0</td>\n",
       "      <td>/nj01hspawPof0mJmlgfjuLyJuRN.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Four Rooms</td>\n",
       "      <td>5.9</td>\n",
       "      <td>2650.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1995-12-09</td>\n",
       "      <td>4257354.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>4000000.0</td>\n",
       "      <td>tt0113101</td>\n",
       "      <td>...</td>\n",
       "      <td>English</td>\n",
       "      <td>Lili Taylor, Valeria Golino, Marisa Tomei, Mar...</td>\n",
       "      <td>Quentin Tarantino, Robert Rodriguez, Allison A...</td>\n",
       "      <td>Andrzej Sekula, Rodrigo García, Guillermo Nava...</td>\n",
       "      <td>Quentin Tarantino, Robert Rodriguez, Allison A...</td>\n",
       "      <td>Quentin Tarantino, Lawrence Bender, Alexandre ...</td>\n",
       "      <td>Combustible Edison</td>\n",
       "      <td>6.7</td>\n",
       "      <td>113089.0</td>\n",
       "      <td>/pyCk5JgtRZwRxnXwfrvyzukaKue.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Judgment Night</td>\n",
       "      <td>6.5</td>\n",
       "      <td>333.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>1993-10-15</td>\n",
       "      <td>12136938.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>21000000.0</td>\n",
       "      <td>tt0107286</td>\n",
       "      <td>...</td>\n",
       "      <td>English</td>\n",
       "      <td>Stephen Dorff, Everlast, Will Zahrn, Emilio Es...</td>\n",
       "      <td>Stephen Hopkins</td>\n",
       "      <td>Peter Levy</td>\n",
       "      <td>Jere Cunningham, Lewis Colick</td>\n",
       "      <td>Gene Levy, Marilyn Vance, Lloyd Segan</td>\n",
       "      <td>Alan Silvestri</td>\n",
       "      <td>6.6</td>\n",
       "      <td>19456.0</td>\n",
       "      <td>/3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>Life in Loops (A Megacities RMX)</td>\n",
       "      <td>7.5</td>\n",
       "      <td>27.0</td>\n",
       "      <td>Released</td>\n",
       "      <td>2006-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>tt0825671</td>\n",
       "      <td>...</td>\n",
       "      <td>English, हिन्दी, 日本語, Pусский, Español</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Timo Novotny</td>\n",
       "      <td>Wolfgang Thaler</td>\n",
       "      <td>Michael Glawogger, Timo Novotny</td>\n",
       "      <td>Ulrich Gehmacher, Timo Novotny</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.2</td>\n",
       "      <td>284.0</td>\n",
       "      <td>/7ln81BRnPR2wqxuITZxEciCe1lc.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                             title  vote_average  vote_count    status  \\\n",
       "0   2                             Ariel           7.1       335.0  Released   \n",
       "1   3               Shadows in Paradise           7.3       400.0  Released   \n",
       "2   5                        Four Rooms           5.9      2650.0  Released   \n",
       "3   6                    Judgment Night           6.5       333.0  Released   \n",
       "4   8  Life in Loops (A Megacities RMX)           7.5        27.0  Released   \n",
       "\n",
       "  release_date     revenue  runtime      budget    imdb_id  ...  \\\n",
       "0   1988-10-21         0.0     73.0         0.0  tt0094675  ...   \n",
       "1   1986-10-17         0.0     74.0         0.0  tt0092149  ...   \n",
       "2   1995-12-09   4257354.0     98.0   4000000.0  tt0113101  ...   \n",
       "3   1993-10-15  12136938.0    109.0  21000000.0  tt0107286  ...   \n",
       "4   2006-01-01         0.0     80.0     42000.0  tt0825671  ...   \n",
       "\n",
       "                         spoken_languages  \\\n",
       "0                                   suomi   \n",
       "1                 English, suomi, svenska   \n",
       "2                                 English   \n",
       "3                                 English   \n",
       "4  English, हिन्दी, 日本語, Pусский, Español   \n",
       "\n",
       "                                                cast  \\\n",
       "0  Heikki Salomaa, Hanna Jokinen, Matti Pellonpää...   \n",
       "1  Teuvo Rissanen, Malla Hukkanen, Olli Varja, Ju...   \n",
       "2  Lili Taylor, Valeria Golino, Marisa Tomei, Mar...   \n",
       "3  Stephen Dorff, Everlast, Will Zahrn, Emilio Es...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                            director  \\\n",
       "0                                     Aki Kaurismäki   \n",
       "1                                     Aki Kaurismäki   \n",
       "2  Quentin Tarantino, Robert Rodriguez, Allison A...   \n",
       "3                                    Stephen Hopkins   \n",
       "4                                       Timo Novotny   \n",
       "\n",
       "                             director_of_photography  \\\n",
       "0                                      Timo Salminen   \n",
       "1                                      Timo Salminen   \n",
       "2  Andrzej Sekula, Rodrigo García, Guillermo Nava...   \n",
       "3                                         Peter Levy   \n",
       "4                                    Wolfgang Thaler   \n",
       "\n",
       "                                             writers  \\\n",
       "0                                     Aki Kaurismäki   \n",
       "1                                     Aki Kaurismäki   \n",
       "2  Quentin Tarantino, Robert Rodriguez, Allison A...   \n",
       "3                      Jere Cunningham, Lewis Colick   \n",
       "4                    Michael Glawogger, Timo Novotny   \n",
       "\n",
       "                                           producers      music_composer  \\\n",
       "0                                     Aki Kaurismäki                 NaN   \n",
       "1                                    Mika Kaurismäki                 NaN   \n",
       "2  Quentin Tarantino, Lawrence Bender, Alexandre ...  Combustible Edison   \n",
       "3              Gene Levy, Marilyn Vance, Lloyd Segan      Alan Silvestri   \n",
       "4                     Ulrich Gehmacher, Timo Novotny                 NaN   \n",
       "\n",
       "  imdb_rating imdb_votes                       poster_path  \n",
       "0         7.4     8870.0  /ojDg0PGvs6R9xYFodRct2kdI6wC.jpg  \n",
       "1         7.5     7654.0  /nj01hspawPof0mJmlgfjuLyJuRN.jpg  \n",
       "2         6.7   113089.0  /pyCk5JgtRZwRxnXwfrvyzukaKue.jpg  \n",
       "3         6.6    19456.0  /3rvvpS9YPM5HB2f4HYiNiJVtdam.jpg  \n",
       "4         8.2      284.0  /7ln81BRnPR2wqxuITZxEciCe1lc.jpg  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import kagglehub\n",
    "\n",
    "# Download latest version\n",
    "path = kagglehub.dataset_download(\"alanvourch/tmdb-movies-daily-updates\")\n",
    "\n",
    "# List files in the downloaded dataset directory\n",
    "files = os.listdir(path)\n",
    "print(\"Files in dataset directory:\", files)\n",
    "\n",
    "# Read the first file\n",
    "updated_df = pd.read_csv(os.path.join(path, files[0]))\n",
    "\n",
    "# Print the first 5 rows\n",
    "updated_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "status\n",
       "Released           1021903\n",
       "Planned               5275\n",
       "Post Production       5179\n",
       "In Production         5060\n",
       "Rumored               1093\n",
       "Canceled               600\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "updated_df['status'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new = updated_df[updated_df['status'] == 'Post Production'].copy()\n",
    "df_new['release_date'] = pd.to_datetime(df_new['release_date'], errors='coerce')\n",
    "df_new.dropna(subset=['release_date'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing the release_date to release_year\n",
    "df_new['release_year'] = pd.to_datetime(df_new['release_date']).dt.year\n",
    "\n",
    "for col in df_new.columns:\n",
    "    if col in df_new.columns:\n",
    "        df_new[col] = df_new[col].where(pd.notna(df_new[col]), None)\n",
    "\n",
    "df_new['timestamp'] = datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmdb_movie_fg = fs.get_or_create_feature_group(\n",
    "    name='tmdb_upcoming_movies',\n",
    "    description='Movie information including director, actors, genres, etc.',\n",
    "    version=5,\n",
    "    primary_key=['id'],\n",
    "    online_enabled=True,\n",
    ")\n",
    "\n",
    "tmdb_movie_fg.insert(df_new)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12 | packaged by conda-forge | (default, Oct 26 2021, 05:59:23) \n[Clang 11.1.0 ]"
  },
  "vscode": {
   "interpreter": {
    "hash": "e76101013b4554409517ea5e48680d10f8b21d3574b21ef810c80afd4c1045a0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}