python-azure-sql/2-insert-data-sql.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python385jvsc74a57bd09ceeaccb2d4b8fc120d52b4319c20d729c878e30972ae0f296691445edd93f69",
   "display_name": "Python 3.8.5 32-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Insert Python dataframe into Azure SQL table"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## Import the libraries"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyodbc\n",
    "import pandas"
   ]
  },
  {
   "source": [
    "## Open the connection"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "server = '<server>.database.windows.net'\n",
    "database = '<database>'\n",
    "username = '<username>'\n",
    "password = '<password>'\n",
    "driver= '{ODBC Driver 17 for SQL Server}'\n",
    "conn = pyodbc.connect('DRIVER=' + driver + ';SERVER=' +\n",
    "    server + ';PORT=1433;DATABASE=' + database +\n",
    "    ';UID=' + username + ';PWD=' + password)"
   ]
  },
  {
   "source": [
    "## Create a cursor object"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor = conn.cursor()"
   ]
  },
  {
   "source": [
    "## Create a table"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop previous table of same name if one exists\n",
    "cursor.execute(\"DROP TABLE IF EXISTS movies\")\n",
    "\n",
    "# Create a table\n",
    "cursor.execute(\"CREATE TABLE movies \"\n",
    "                \"(movie_id INT IDENTITY(1,1) PRIMARY KEY, \"\n",
    "                \"movie_title VARCHAR(255) NOT NULL, \"\n",
    "                \"released_year INT, \"\n",
    "                \"runtime INT NOT NULL, \"\n",
    "                \"genre VARCHAR(255), \"\n",
    "                \"rating INT NOT NULL, \"\n",
    "                \"director VARCHAR(255), \"\n",
    "                \"star1 VARCHAR(255), \"\n",
    "                \"star2 VARCHAR(255), \"\n",
    "                \"number_of_votes INT, \"\n",
    "                \"gross INT)\")"
   ]
  },
  {
   "source": [
    "## Open the file"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpath = \"movies.CSV\"\n",
    "df = pandas.read_csv(fpath, delimiter=\";\", encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "source": [
    "## Transform columns and change data types"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### Runtime"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_runtime(x):\n",
    "    \"\"\"\n",
    "    Convert the string value to int.\n",
    "    - Remove \"min\".\n",
    "    - Convert to int.\n",
    "    \"\"\"\n",
    "    return int(x.split()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Runtime'] = df['Runtime'].apply(convert_runtime)"
   ]
  },
  {
   "source": [
    "### Genre"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Genre'] = df['Genre'].apply(lambda x: x.split(', ')[0])"
   ]
  },
  {
   "source": [
    "### Gross"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_gross(x):\n",
    "    \"\"\"\n",
    "    Convert the string value to int.\n",
    "    - Remove commas.\n",
    "    - Convert to int.\n",
    "    \"\"\"\n",
    "    return int(x.replace(',', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Gross'] = df['Gross'].apply(convert_gross)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "source": [
    "## Insert data in the *movies* table"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df.iterrows():\n",
    "    cursor.execute(\"INSERT INTO movies \"\n",
    "                   \"(movie_title, released_year, runtime, \"\n",
    "                   \"genre, rating, director, \"\n",
    "                   \"star1, star2, number_of_votes, gross) \"\n",
    "                   \"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\", \n",
    "                   (row['Title'], row['Released_Year'], row['Runtime'], row['Genre'], row['Rating'], \n",
    "                   row['Director'], row['Star1'], row['Star2'], row['Number_of_Votes'], row['Gross']))"
   ]
  },
  {
   "source": [
    "## Commit the changes"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn.commit()"
   ]
  },
  {
   "source": [
    "## Read data"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Read data\n",
    "cursor.execute(\"SELECT \"\n",
    "    \"movie_id, movie_title \"\n",
    "    \"FROM movies\")\n",
    "rows = cursor.fetchall()\n",
    "\n",
    "# Print data\n",
    "for row in rows:\n",
    "  print(row)"
   ]
  },
  {
   "source": [
    "## Close the connection"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor.close()\n",
    "conn.close()"
   ]
  }
 ]
}