Roko_Ch_10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "89451c33",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The decision boundary of each output neuron is linear, so perceptrons are\n",
    "#incapable of learning complex patterns (just like logistic regression\n",
    "#classifiers). However, if the training instances are linearly separable, Rosenblatt demonstrated \n",
    "#that this algorithm would converge to a solution.7\n",
    "#This is called the perceptron convergence theorem.\n",
    "#Scikit-Learn provides a Perceptron class that can be used pretty much as\n",
    "#you would expect—for example, on the iris dataset (introduced in Chapter 4):\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "\n",
    "iris = load_iris(as_frame=True)\n",
    "X = iris.data[[\"petal length (cm)\", \"petal width (cm)\"]].values\n",
    "y = (iris.target == 0) #Iris setosa\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f9c66437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Perceptron(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Perceptron</label><div class=\"sk-toggleable__content\"><pre>Perceptron(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "Perceptron(random_state=42)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "per_clf = Perceptron(random_state=42)\n",
    "per_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c67a2a4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new = [[2, 0.5], [3, 1]]\n",
    "y_pred = per_clf.predict(X_new)\n",
    "y_pred ## predicts True and False for these 2 flowers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "369b095d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Contrary to logistic regression classifiers, perceptrons do not output a class probability. This is one \n",
    "#reason to prefer logistic regression over perceptrons. Moreover, perceptrons do not use any regularization\n",
    "#by default, and training stops as soon as there are no more prediction errors on the training set, so the\n",
    "#model typically does not generalize as well as logistic regression or a linear SVM classifier. However,\n",
    "#perceptrons may train a bit faster.\n",
    "\n",
    "\n",
    "\n",
    "#The Multilayer Perceptron and Backpropagation\n",
    "#An MLP is composed of one input layer, one or more layers of TLUs called hidden layers, and one \n",
    "#final layer of TLUs called the output layer (see Figure 10-7). The layers close to the input layer\n",
    "#are usually called the lower layers, and the ones close to the outputs are usually called the upper layers.\n",
    "\n",
    "\n",
    "\n",
    "#The signal flows only in one direction (from the inputs to the outputs), so this architecture\n",
    "#is an example of a feedforward neural network (FNN)\n",
    "\n",
    "\n",
    "#When an ANN contains a deep stack of hidden layers,9 it is called a deep neural network (DNN). \n",
    "#The field of deep learning studies DNNs, and more generally it is interested in models containing\n",
    "#deep stacks of computations\n",
    "\n",
    "#Then, in 1970, a researcher named Seppo Linnainmaa introduced in his master’s thesis a technique to \n",
    "#compute all the gradients automatically and efficiently. This algorithm is now called reverse-mode \n",
    "#automatic differentiation (or reverse-mode autodiff for short). In just two passes through the network\n",
    "#(one forward, one backward), it is able to compute the gradients of the neural network’s error with\n",
    "#regard to every single model parameter. In other words, it can find out how each connection weight\n",
    "#and each bias should be tweaked in order to reduce the neural network’s error. These gradients can\n",
    "#then be used to perform a gradient descent step. If you repeat this process of computing the gradients\n",
    "#automatically and taking a gradient descent step, the neural network’s error will gradually drop\n",
    "#until it eventually reaches a minimum. This combination of reverse-mode autodiff and gradient \n",
    "#descent is now called backpropagation (or backprop for short).\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#It is important to initialize all the hidden layers’ connection weights randomly, or else training will fail.\n",
    "#For example, if you initialize all weights and biases to zero, then all neurons in a given layer will be\n",
    "#perfectly identical, and thus backpropagation will affect them in exactly the same way, so they will remain\n",
    "#identical. In other words, despite having hundreds of neurons per layer, your model will act as if it had\n",
    "#only one neuron per layer: it won’t be too smart. If instead you randomly initialize the weights, you \n",
    "#break the symmetry and allow backpropagation to train a diverse team of neurons.\n",
    "\n",
    "\n",
    "\n",
    "#In short, backpropagation makes predictions for a mini-batch (forward pass), measures the error, \n",
    "#then goes through each layer in reverse to measure the error contribution from each parameter \n",
    "#(reverse pass), and finally tweaks the connection weights and biases to reduce the error (gradient descent step).\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#In order for backprop to work properly, Rumelhart and his colleagues made a key change to the MLP’s \n",
    "#architecture: they replaced the step function with the logistic function, σ(z) = 1 / (1 + exp(–z)), \n",
    "#also called the sigmoid function. This was essential because the step function contains only flat \n",
    "#segments, so there is no gradient to work with (gradient descent cannot move on a flat surface),\n",
    "#while the sigmoid function has a well-defined nonzero derivative everywhere, allowing gradient \n",
    "#descent to make some progress at every step. In fact, the backpropagation algorithm works well \n",
    "#with many other activation functions, not just the sigmoid function. Here are two other popular choices:\n",
    "\n",
    "\n",
    "#These popular activation functions and their derivatives are represented in Figure 10-8. But wait!\n",
    "#Why do we need activation functions in the first place? Well, if you chain several linear transformations, \n",
    "#all you get is a linear transformation. For example, if f(x) = 2x + 3 and g(x) = 5x – 1, then chaining \n",
    "#these two linear functions gives you another linear function: f(g(x)) = 2(5x – 1) + 3 = 10x + 1. So if \n",
    "#you don’t have some nonlinearity between layers, then even a deep stack of layers is equivalent to a \n",
    "#single layer, and you can’t solve very complex problems with that. Conversely, a large enough DNN with\n",
    "#nonlinear activations can theoretically approximate any continuous function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "953bde45",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Regression MLPs\n",
    "#First, MLPs can be used for regression tasks. If you want to predict a single value (e.g., the price of a house,\n",
    "#given many of its features), then you just need a single output neuron: its output is the predicted value. For\n",
    "#multivariate regression (i.e., to predict multiple values at once), you need one output neuron per output \n",
    "#dimension. For example, to locate the center of an object in an image, you need to predict 2D coordinates, \n",
    "#so you need two output neurons. If you also want to place a bounding box around the object, then you need\n",
    "#two more numbers: the width and the height of the object. So, you end up with four output neurons.\n",
    "\n",
    "#Scikit-Learn includes an MLPRegressor class, so let’s use it to build an MLP with three hidden layers composed \n",
    "#of 50 neurons each, and train it on the California housing dataset. For simplicity, we will use Scikit-Learn’s\n",
    "#fetch_california_housing() function to load the data. This dataset is simpler than the one we used in Chapter 2, \n",
    "#since it contains only numerical features (there is no ocean_proximity feature), and there are no missing values.\n",
    "#The following code starts by fetching and splitting the dataset, then it creates a pipeline to standardize the \n",
    "#input features before sending them to the MLPRegressor. This is very important for neural networks because they\n",
    "#are trained using gradient descent, and as we saw in Chapter 4, gradient descent does not converge very well \n",
    "#when the features have very different scales. Finally, the code trains the model and evaluates its validation\n",
    "#error. The model uses the ReLU activation function in the hidden layers, and it uses a variant of gradient \n",
    "#descent called Adam (see Chapter 11) to minimize the mean squared error, with a little bit of l2 regularization\n",
    "#(which you can control via the alpha hyperparameter):\n",
    "\n",
    "\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state=42)\n",
    "\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "\n",
    "mlp_reg = MLPRegressor(hidden_layer_sizes=[50,50,50], random_state=42)\n",
    "pipeline = make_pipeline(StandardScaler(), mlp_reg)\n",
    "pipeline.fit(X_train, y_train)\n",
    "y_pred = pipeline.predict(X_valid)\n",
    "\n",
    "rmse = mean_squared_error(y_valid, y_pred, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "727e59e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5053326657968719"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e5230cba",
   "metadata": {},
   "outputs": [],
   "source": [
    "#We get a validation RMSE of about 0.505, which is comparable to what you\n",
    "#would get with a random forest classifier. Not too bad for a first try!\n",
    "#Note that this MLP does not use any activation function for the output layer, so it’s free to output any \n",
    "#value it wants. This is generally fine, but if you want to guarantee that the output will always be positive,\n",
    "#then you should use the ReLU activation function in the output layer, or the softplus activation function,\n",
    "#which is a smooth variant of ReLU: softplus(z) = log(1 + exp(z)). Softplus is close to 0 when z is negative,\n",
    "#and close to z when z is positive. Finally, if you want to guarantee that the predictions will always fall\n",
    "#within a given range of values, then you should use the sigmoid function or the hyperbolic tangent, and scale \n",
    "#the targets to the appropriate range: 0 to 1 for sigmoid and –1 to 1 for tanh. Sadly, the MLPRegressor class \n",
    "#does not support activation functions in the output layer.\n",
    "\n",
    "\n",
    "#Building and training a standard MLP with Scikit-Learn in just a few lines of code is very convenient, but \n",
    "#the neural net features are limited. This is why we will switch to Keras in the second part of this chapter.\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "565bfef4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The MLPRegressor class uses the mean squared error, which is usually what you want for regression, but \n",
    "#if you have a lot of outliers in the training set, you may prefer to use the mean absolute error instead. \n",
    "#Alternatively, you may want to use the Huber loss, which is a combination of both. It is quadratic when \n",
    "#the error is smaller than a threshold δ (typically 1) but linear when the error is larger than δ. The \n",
    "#linear part makes it less sensitive to outliers than the mean squared error, and the quadratic part allows \n",
    "#it to converge faster and be more precise than the mean absolute error. However, MLPRegressor only supports the MSE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "71b6cd73",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Classification MLPs\n",
    "#MLPs can also be used for classification tasks. For a binary classification problem, you just need a single \n",
    "#output neuron using the sigmoid activation function: the output will be a number between 0 and 1, which you \n",
    "#can interpret as the estimated probability of the positive class. The estimated probability of the negative \n",
    "#class is equal to one minus that number. MLPs can also easily handle multilabel binary classification tasks \n",
    "#(see Chapter 3). For example, you could have an email classification system that predicts whether each \n",
    "#incoming email is ham or spam, and simultaneously predicts whether it is an urgent or nonurgent email. \n",
    "#In this case, you would need two output neurons, both using the sigmoid activation function: the first would \n",
    "#output the probability that the email is spam, and the second would output the probability that it is urgent.\n",
    "#More generally, you would dedicate one output neuron for each positive class. Note that the output probabilities\n",
    "#do not necessarily add up to 1. This lets the model output any combination of labels: you can have nonurgent\n",
    "#ham, urgent ham, nonurgent spam, and perhaps even urgent spam (although that would probably be an error).\n",
    "\n",
    "#If each instance can belong only to a single class, out of three or more possible classes (e.g., classes 0 \n",
    "#through 9 for digit image classification), then you need to have one output neuron per class, and you should\n",
    "#use the softmax activation function for the whole output layer (see Figure 10-9). The softmax function (introduced \n",
    "#in Chapter 4) will ensure that all the estimated probabilities are between 0 and 1 and that they add up to 1,\n",
    "#since the classes are exclusive. As you saw in Chapter 3, this is called multiclass classification.\n",
    "\n",
    "\n",
    "#Regarding the loss function, since we are predicting probability distributions, the cross-entropy loss \n",
    "#(or x-entropy or log loss for short, see Chapter 4) is generally a good choice.\n",
    "\n",
    "\n",
    "#Scikit-Learn has an MLPClassifier class in the sklearn.neural_network package. It is almost identical to \n",
    "#the MLPRegressor class, except that it minimizes the cross entropy rather than the MSE. Give it a try now,\n",
    "#for example on the iris dataset. It’s almost a linear task, so a single layer with 5 to 10 neurons should \n",
    "#suffice (make sure to scale the features).\n",
    "\n",
    "\n",
    "\n",
    "#Scikit-Learn has an MLPClassifier class in the sklearn.neural_network package. It is almost identical to the\n",
    "#MLPRegressor class, except that it minimizes the cross entropy rather than the MSE. Give it a try now, for \n",
    "#example on the iris dataset. It’s almost a linear task, so a single layer with 5 to 10 neurons should suffice\n",
    "#(make sure to scale the features).\n",
    "\n",
    "\n",
    "#TIP\n",
    "#Before we go on, I recommend you go through exercise 1 at the end of this chapter. You will play with \n",
    "#various neural network architectures and visualize their outputs using the TensorFlow playground. This will\n",
    "#be very useful to better understand MLPs, including the effects of all the hyperparameters (number of layers \n",
    "#and neurons, activation functions, and more)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a3e3e6bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Implementing MLPs with Keras\n",
    "#Keras is TensorFlow’s high-level deep learning API: it allows you to build, train, \n",
    "#evaluate, and execute all sorts of neural networks. The original Keras library was \n",
    "#developed by François Chollet as part of a research project12 and was released as \n",
    "#a standalone open source project in March 2015. It quickly gained popularity, owing\n",
    "#to its ease of use, flexibility, and beautiful design.\n",
    "\n",
    "\n",
    "\n",
    "#Now let’s use Keras! We will start by building an MLP for image classification.\n",
    "#Building an Image Classifier Using the Sequential API\n",
    "#First, we need to load a dataset. We will use Fashion MNIST, which is a drop-in \n",
    "#replacement of MNIST (introduced in Chapter 3). It has the exact same format as \n",
    "#MNIST (70,000 grayscale images of 28 × 28 pixels each, with 10 classes), but the \n",
    "#images represent fashion items rather than handwritten digits, so each class is \n",
    "#more diverse, and the problem turns out to be significantly more challenging than \n",
    "#MNIST. For example, a simple linear model reaches about 92% accuracy on MNIST, but\n",
    "#only about 83% on Fashion MNIST.\n",
    "\n",
    "\n",
    "\n",
    "#Building an Image Classifier Using the Sequential API\n",
    "#First, we need to load a dataset. We will use Fashion MNIST, which is a drop-in \n",
    "#replacement of MNIST (introduced in Chapter 3). It has the exact same format as\n",
    "#MNIST (70,000 grayscale images of 28 × 28 pixels each, with 10 classes), but the \n",
    "#images represent fashion items rather than handwritten digits, so each class is more\n",
    "#diverse, and the problem turns out to be significantly more challenging than MNIST. \n",
    "#For example, a simple linear model reaches about 92% accuracy on MNIST, but only \n",
    "#about 83% on Fashion MNIST.\n",
    "\n",
    "\n",
    "#Using Keras to load the dataset\n",
    "#Keras provides some utility functions to fetch and load common datasets, including MNIST,\n",
    "#Fashion MNIST, and a few more. Let’s load Fashion MNIST. It’s already shuffled and split\n",
    "#into a training set (60,000 images) and a test set (10,000 images), but we’ll hold out \n",
    "#the last 5,000 images from the training set for validation:\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "fashion_mnist = tf.keras.datasets.fashion_mnist.load_data()\n",
    "(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist\n",
    "X_train, y_train = X_train_full[:-5000], y_train_full[:-5000]\n",
    "X_valid, y_valid = X_train_full[-5000:], y_train_full[-5000:]\n",
    "\n",
    "#TensorFlow is usually imported as tf, and the Keras API is available via tf.keras."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d111269b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((55000, 28, 28), dtype('uint8'))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#When loading MNIST or Fashion MNIST using Keras rather than Scikit- Learn, one \n",
    "#important difference is that every image is represented as a 28 × 28 array rather than \n",
    "#a 1D array of size 784. Moreover, the pixel intensities are represented as integers \n",
    "#(from 0 to 255) rather than floats (from 0.0 to 255.0). Let’s take a look at the \n",
    "#shape and data type of the training set:\n",
    "\n",
    "X_train.shape, X_train.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "838d39ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#For simplicity, we’ll scale the pixel intensities down to the 0–1 range by dividing them\n",
    "#by 255.0 (this also converts them to floats):\n",
    "\n",
    "X_train, X_valid, X_test = X_train / 255., X_valid / 255., X_test / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eace2abe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Ankle boot'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#With MNIST, when the label is equal to 5, it means that the image represents the \n",
    "#handwritten digit 5. Easy. For Fashion MNIST, however, we need the list of class \n",
    "#names to know what we are dealing with:\n",
    "\n",
    "class_names = [\"T-shirt/top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\",\n",
    "              \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle boot\"]\n",
    "\n",
    "#For example, the first image in the training set represents an ankle boot:\n",
    "\n",
    "class_names[y_train[0]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c98e9682",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metal device set to: Apple M1 Pro\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:24:44.663223: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2023-03-31 10:24:44.663504: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    }
   ],
   "source": [
    "#Creating the model using the sequential API\n",
    "#Now let’s build the neural network! Here is a classification MLP with two hidden layers:\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Input(shape=[28, 28]))\n",
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(300, activation=\"relu\"))\n",
    "model.add(tf.keras.layers.Dense(100, activation=\"relu\"))\n",
    "model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "20516f8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " flatten_1 (Flatten)         (None, 784)               0         \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 300)               235500    \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 100)               30100     \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 10)                1010      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Instead of adding the layers one by one as we just did, it’s often more convenient \n",
    "#to pass a list of layers when creating the Sequential model. You can also drop the \n",
    "#Input layer and instead specify the input_shape in the first layer:\n",
    "\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=[28,28]),\n",
    "    tf.keras.layers.Dense(300, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#The model’s summary() method displays all the model’s layers,14 including each layer’s \n",
    "#name (which is automatically generated unless you set it when creating the layer), \n",
    "#its output shape (None means the batch size can be anything), and its number of \n",
    "#parameters. The summary ends with the total number of parameters, including \n",
    "#trainable and non-trainable parameters. Here we only have trainable parameters \n",
    "#(you will see some non-trainable parameters later in this chapter):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fd1c8429",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.layers.reshaping.flatten.Flatten at 0x17c9b1660>,\n",
       " <keras.layers.core.dense.Dense at 0x139ae3520>,\n",
       " <keras.layers.core.dense.Dense at 0x139ae3a60>,\n",
       " <keras.layers.core.dense.Dense at 0x139ae1fc0>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Note that Dense layers often have a lot of parameters. For example, the first hidden layer \n",
    "#has 784 × 300 connection weights, plus 300 bias terms, which adds up to 235,500 parameters!\n",
    "#This gives the model quite a lot of flexibility to fit the training data, but it also means\n",
    "#that the model runs the risk of overfitting, especially when you do not have a lot of \n",
    "#training data. We will come back to this later.\n",
    "\n",
    "#Each layer in a model must have a unique name (e.g., \"dense_2\"). You can set the layer \n",
    "#names explicitly using the constructor’s name argument, but generally it’s simpler to \n",
    "#let Keras name the layers automatically, as we just did. Keras takes the layer’s class\n",
    "#name and converts it to snake case (e.g., a layer from the MyCoolLayer class is named \n",
    "#\"my_cool_layer\" by default). Keras also ensures that the name is globally unique, even \n",
    "#across models, by appending an index if needed, as in \"dense_2\". But why does it bother\n",
    "#making the names unique across models? Well, this makes it possible to merge models \n",
    "#easily without getting name conflicts.\n",
    "\n",
    "#You can easily get a model’s list of layers using the layers attribute, or use \n",
    "#the get_layer() method to access a layer by name:\n",
    "\n",
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3821eba7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dense_3'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden1 = model.layers[1]\n",
    "hidden1.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1041fc30",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No such layer: dense_12. Existing layers are: ['flatten_1', 'dense_3', 'dense_4', 'dense_5'].",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdense_12\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mis\u001b[39;00m hidden1\n",
      "File \u001b[0;32m~/miniforge3/envs/homl3/lib/python3.10/site-packages/keras/engine/training.py:3275\u001b[0m, in \u001b[0;36mModel.get_layer\u001b[0;34m(self, name, index)\u001b[0m\n\u001b[1;32m   3273\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m layer\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m==\u001b[39m name:\n\u001b[1;32m   3274\u001b[0m             \u001b[38;5;28;01mreturn\u001b[39;00m layer\n\u001b[0;32m-> 3275\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   3276\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo such layer: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Existing layers are: \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3277\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlist\u001b[39m(layer\u001b[38;5;241m.\u001b[39mname \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3278\u001b[0m     )\n\u001b[1;32m   3279\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   3280\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProvide either a layer name or layer index at \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`get_layer`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3281\u001b[0m )\n",
      "\u001b[0;31mValueError\u001b[0m: No such layer: dense_12. Existing layers are: ['flatten_1', 'dense_3', 'dense_4', 'dense_5']."
     ]
    }
   ],
   "source": [
    "model.get_layer('dense_12') is hidden1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "625a2a3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.dense.Dense at 0x139ae3520>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7e20f600",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00606299,  0.06024863, -0.04134121, ...,  0.01615225,\n",
       "         0.04387075, -0.06343479],\n",
       "       [-0.07429163, -0.03110987, -0.04809699, ...,  0.0559269 ,\n",
       "         0.02748938, -0.04144006],\n",
       "       [ 0.0055501 , -0.04907951, -0.06057887, ..., -0.03036299,\n",
       "        -0.01647884,  0.05967766],\n",
       "       ...,\n",
       "       [-0.05988706,  0.06762692,  0.07248458, ..., -0.06857683,\n",
       "         0.01020861,  0.00685229],\n",
       "       [-0.04225935,  0.06826448, -0.02383086, ...,  0.04170898,\n",
       "         0.05829886, -0.04404605],\n",
       "       [-0.03183773,  0.0661739 , -0.0593466 , ..., -0.05865016,\n",
       "         0.01461947, -0.04097207]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#All the parameters of a layer can be accessed using its get_weights() and set_weights() \n",
    "#methods. For a Dense layer, this includes both the connection weights and the bias terms:\n",
    "\n",
    "weights, biases = hidden1.get_weights()\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "62914737",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((784, 300), (300,))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights.shape, biases.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1d8b142a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Notice that the Dense layer initialized the connection weights randomly (which is needed \n",
    "#to break symmetry, as discussed earlier), and the biases were initialized to zeros, which \n",
    "#is fine. If you want to use a different initialization method, you can set \n",
    "#kernel_initializer (kernel is another name for the matrix of connection weights) \n",
    "#or bias_initializer when creating the layer. We’ll discuss initializers further \n",
    "#in Chapter 11, and the full list is at https://keras.io/api/layers/initializers.\n",
    "\n",
    "#Compiling the model\n",
    "#After a model is created, you must call its compile() method to specify the loss \n",
    "#function and the optimizer to use. Optionally, you can specify a list of extra metrics\n",
    "#to compute during training and evaluation:\n",
    "\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "             optimizer=\"sgd\",\n",
    "             metrics=[\"accuracy\"])\n",
    "\n",
    "#This code requires explanation. We use the \"sparse_categorical_crossentropy\" loss \n",
    "#because we have sparse labels (i.e., for each instance, there is just a target class\n",
    "#index, from 0 to 9 in this case), and the classes are exclusive. If instead we had\n",
    "#one target probability per class for each instance (such as one-hot vectors, e.g., \n",
    "#[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.] to represent class 3), then we would \n",
    "#need to use the \"categorical_crossentropy\" loss instead. If we were doing binary \n",
    "#classification or multilabel binary classification, then we would use the \"sigmoid\"\n",
    "#activation function in the output layer instead of the \"softmax\" activation function,\n",
    "#and we would use the \"binary_crossentropy\" loss.\n",
    "\n",
    "\n",
    "#Regarding the optimizer, \"sgd\" means that we will train the model using stochastic \n",
    "#gradient descent. In other words, Keras will perform the backpropagation algorithm \n",
    "#described earlier (i.e., reverse-mode autodiff plus gradient descent). We will discuss\n",
    "#more efficient optimizers in Chapter 11. They improve gradient descent, not autodiff.\n",
    "\n",
    "#NOTE\n",
    "#When using the SGD optimizer, it is important to tune the learning rate. So, you \n",
    "#will generally want to use optimizer=tf.keras. optimizers.SGD(learning_rate=__???__)\n",
    "#to set the learning rate, rather than optimizer=\"sgd\", which defaults to a learning \n",
    "#rate of 0.01\n",
    "\n",
    "\n",
    "#Finally, since this is a classifier, it’s useful to measure its accuracy during \n",
    "#training and evaluation, which is why we set metrics=[\"accuracy\"]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "11d2d1c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:24:57.030473: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-03-31 10:24:57.150957: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1719/1719 [==============================] - ETA: 0s - loss: 0.7224 - accuracy: 0.7612"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:25:05.030723: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1719/1719 [==============================] - 9s 5ms/step - loss: 0.7224 - accuracy: 0.7612 - val_loss: 0.5098 - val_accuracy: 0.8270\n",
      "Epoch 2/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.4879 - accuracy: 0.8297 - val_loss: 0.4657 - val_accuracy: 0.8332\n",
      "Epoch 3/30\n",
      "1719/1719 [==============================] - 9s 5ms/step - loss: 0.4413 - accuracy: 0.8447 - val_loss: 0.4239 - val_accuracy: 0.8502\n",
      "Epoch 4/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.4158 - accuracy: 0.8549 - val_loss: 0.3979 - val_accuracy: 0.8606\n",
      "Epoch 5/30\n",
      "1719/1719 [==============================] - 9s 5ms/step - loss: 0.3952 - accuracy: 0.8608 - val_loss: 0.3918 - val_accuracy: 0.8598\n",
      "Epoch 6/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3788 - accuracy: 0.8664 - val_loss: 0.4004 - val_accuracy: 0.8628\n",
      "Epoch 7/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3662 - accuracy: 0.8704 - val_loss: 0.3736 - val_accuracy: 0.8666\n",
      "Epoch 8/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3544 - accuracy: 0.8744 - val_loss: 0.3768 - val_accuracy: 0.8608\n",
      "Epoch 9/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3437 - accuracy: 0.8787 - val_loss: 0.3519 - val_accuracy: 0.8718\n",
      "Epoch 10/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3337 - accuracy: 0.8809 - val_loss: 0.3535 - val_accuracy: 0.8720\n",
      "Epoch 11/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3248 - accuracy: 0.8852 - val_loss: 0.3660 - val_accuracy: 0.8654\n",
      "Epoch 12/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3169 - accuracy: 0.8861 - val_loss: 0.3486 - val_accuracy: 0.8714\n",
      "Epoch 13/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3097 - accuracy: 0.8901 - val_loss: 0.3300 - val_accuracy: 0.8792\n",
      "Epoch 14/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.3025 - accuracy: 0.8919 - val_loss: 0.3408 - val_accuracy: 0.8780\n",
      "Epoch 15/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2967 - accuracy: 0.8948 - val_loss: 0.3381 - val_accuracy: 0.8798\n",
      "Epoch 16/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2894 - accuracy: 0.8957 - val_loss: 0.3305 - val_accuracy: 0.8814\n",
      "Epoch 17/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2847 - accuracy: 0.8979 - val_loss: 0.3387 - val_accuracy: 0.8752\n",
      "Epoch 18/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2785 - accuracy: 0.8999 - val_loss: 0.3272 - val_accuracy: 0.8818\n",
      "Epoch 19/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2733 - accuracy: 0.9013 - val_loss: 0.3563 - val_accuracy: 0.8672\n",
      "Epoch 20/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2688 - accuracy: 0.9041 - val_loss: 0.3203 - val_accuracy: 0.8824\n",
      "Epoch 21/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2640 - accuracy: 0.9050 - val_loss: 0.3150 - val_accuracy: 0.8846\n",
      "Epoch 22/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2584 - accuracy: 0.9063 - val_loss: 0.3130 - val_accuracy: 0.8856\n",
      "Epoch 23/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2542 - accuracy: 0.9084 - val_loss: 0.3391 - val_accuracy: 0.8760\n",
      "Epoch 24/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2491 - accuracy: 0.9103 - val_loss: 0.3211 - val_accuracy: 0.8798\n",
      "Epoch 25/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2449 - accuracy: 0.9118 - val_loss: 0.3172 - val_accuracy: 0.8836\n",
      "Epoch 26/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2419 - accuracy: 0.9139 - val_loss: 0.3151 - val_accuracy: 0.8852\n",
      "Epoch 27/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2365 - accuracy: 0.9152 - val_loss: 0.3182 - val_accuracy: 0.8824\n",
      "Epoch 28/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2339 - accuracy: 0.9161 - val_loss: 0.3154 - val_accuracy: 0.8860\n",
      "Epoch 29/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2295 - accuracy: 0.9180 - val_loss: 0.3209 - val_accuracy: 0.8840\n",
      "Epoch 30/30\n",
      "1719/1719 [==============================] - 8s 5ms/step - loss: 0.2261 - accuracy: 0.9195 - val_loss: 0.3103 - val_accuracy: 0.8880\n"
     ]
    }
   ],
   "source": [
    "#Training and evaluating the model\n",
    "#Now the model is ready to be trained. For this we simply need to call its fit() method:\n",
    "\n",
    "history = model.fit(X_train, y_train, epochs=30, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d19550b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#We pass it the input features (X_train) and the target classes (y_train), as well as \n",
    "#the number of epochs to train (or else it would default to just 1, which would definitely\n",
    "#not be enough to converge to a good solution). We also pass a validation set (this is \n",
    "#optional). Keras will measure the loss and the extra metrics on this set at the end of\n",
    "#each epoch, which is very useful to see how well the model really performs. If the \n",
    "#performance on the training set is much better than on the validation set, your model\n",
    "#is probably overfitting the training set, or there is a bug, such as a data mismatch \n",
    "#between the training set and the validation set.\n",
    "\n",
    "#And that’s it! The neural network is trained. At each epoch during training, Keras \n",
    "#displays the number of mini-batches processed so far on the left side of the progress\n",
    "#bar. The batch size is 32 by default, and since the training set has 55,000 images, \n",
    "#the model goes through 1,719 batches per epoch: 1,718 of size 32, and 1 of size 24.\n",
    "#After the progress bar, you can see the mean training time per sample, and the \n",
    "#loss and accuracy (or any other extra metrics you asked for) on both the training \n",
    "#set and the validation set. Notice that the training loss went down, which is a good \n",
    "#sign, and the validation accuracy reached 88.94% after 30 epochs. That’s slightly \n",
    "#below the training accuracy, so there is a little bit of overfitting going on, but\n",
    "#not a huge amount.\n",
    "\n",
    "\n",
    "#Instead of passing a validation set using the validation_data argument, you could \n",
    "#set validation_split to the ratio of the training set that you want Keras to use for \n",
    "#validation. For example, validation_split=0.1 tells Keras to use the last 10% of the data\n",
    "#(before shuffling) for validation.\n",
    "\n",
    "\n",
    "\n",
    "#If the training set was very skewed, with some classes being \n",
    "#overrepresented and others underrepresented, it would be useful to set\n",
    "#the class_weight argument when calling the fit() method, to give a \n",
    "#larger weight to underrepresented classes and a lower weight to\n",
    "#overrepresented classes. These weights would be used by Keras when \n",
    "#computing the loss. If you need per-instance weights, set the \n",
    "#sample_weight argument. If both class_weight and sample_weight are\n",
    "#provided, then Keras multiplies them. Per-instance weights could be\n",
    "#useful, for example, if some instances were labeled by experts while \n",
    "#others were labeled using a crowdsourcing platform: you might want \n",
    "#to give more weight to the former. You can also provide sample weights \n",
    "#(but not class weights) for the validation set by adding them as a \n",
    "#third item in the validation_data tuple.\n",
    "\n",
    "\n",
    "#The fit() method returns a History object containing the training \n",
    "#parameters (history.params), the list of epochs it went through \n",
    "#(history.epoch), and most importantly a dictionary (history.history) \n",
    "#containing the loss and extra metrics it measured at the end of each\n",
    "#epoch on the training set and on the validation set (if any). If you\n",
    "#use this dictionary to create a Pandas DataFrame and call its plot()\n",
    "#method, you get the learning curves shown in Figure 10-11:\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "941945d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp8AAAHFCAYAAAC0FZIEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACCgElEQVR4nO3dd3xTZd8G8CtN96QDOmjZIHu1bNkyFRBQQZGhICBLRKY8KuBAfR+GijgBHxUREQdqFVCLslQoFBHKHmUUSwt00jZtzvvHj9PM7pGmvb6fzyHpycnJ3ZyGXr2nRlEUBUREREREFcDB1gUgIiIiouqD4ZOIiIiIKgzDJxERERFVGIZPIiIiIqowDJ9EREREVGEYPomIiIiowjB8EhEREVGFYfgkIiIiogrD8ElEREREFYbhk4iIiIgqTLHD5++//44hQ4YgJCQEGo0G33zzTaHP+e233xAeHg5XV1c0aNAA7777bknKSkRERER2rtjhMz09HW3atMGaNWuKdPz58+cxePBgdO/eHYcPH8azzz6LWbNmYevWrcUuLBERERHZN42iKEqJn6zR4Ouvv8b999+f7zELFizAtm3bEBsbm7dv6tSpOHLkCPbv31/SlyYiIiIiO+RY3i+wf/9+9O/f32TfgAEDsG7dOuh0Ojg5OVk8JysrC1lZWXlf6/V63LhxA/7+/tBoNOVdZCIiIiIqJkVRkJqaipCQEDg45N+4Xu7h89q1awgMDDTZFxgYiJycHCQmJiI4ONjiOcuXL8fSpUvLu2hEREREVMYuXbqE0NDQfB8v9/AJwKK2Um3pz68Wc9GiRZgzZ07e18nJyahTpw5OnToFPz+/8isolZpOp0NUVBR69+5ttVabKg9eK/vBa2UfeJ3sB69V+UhNTUX9+vXh5eVV4HHlHj6DgoJw7do1k30JCQlwdHSEv7+/1ee4uLjAxcXFYr+fn1++z6HKQafTwd3dHf7+/vxAV3K8VvaD18o+8DrZD16r8qG+l4V1kSz3eT67dOmCnTt3muzbsWMHIiIieMGJiIiIqplih8+0tDTExMQgJiYGgEylFBMTg7i4OADSZD5u3Li846dOnYqLFy9izpw5iI2Nxfr167Fu3TrMnTu3bL4DIiIiIrIbxW52P3jwIHr37p33tdo3c/z48fjoo48QHx+fF0QBoH79+oiMjMTTTz+Nt99+GyEhIXjzzTcxcuTIMig+EREREdmTYofPXr16oaCpQT/66COLfT179sShQ4eK+1JEREREVMVwbXciIiIiqjAMn0RERERUYRg+iYiIiKjCMHwSERERUYVh+CQiIiKiCsPwSUREREQVhuGTiIiIiCoMwycRERERVRiGTyIiIiKqMAyfRERERFRhGD6JiIiIqMIwfBIRERFRhWH4JCIiIqIKw/BJRERERBWG4ZOIiIiIKgzDJxERERFVGIZPIiIiIqowDJ9EREREVGEYPomIiIiowjB8EhEREVGFYfgkIiIiogrD8ElERERUnVy+DERFyW1ZunKlSIcxfBIRERGVhfIIdWV9znXrgLp1gT595HbduoKPz80FMjKAmzeB27cN+9PSgOhoYN8+YNcu4OmngRYtilQEx5KXnoiIiMgOXb6MgKNHgdatgfr1y+ac69YBkycDej3g4AC8/z4wcWLpzvnBB8DUqZbnvHEDOHUKyMyUQGh+27cv0LSpnOPvv4H33pPHkpKAb781nF+vlzIPGACEhgK//w489BCQlSVbdraET9XKlRIy1fN261aib4vhk4iIiMrG5cvA6dNA48YSZirLORUF0OmkBm/dOjjOn49uej2UF16QQNe3L3D2rByTnW253X8/UKuWnGvvXuDHH00fv3kT+Pxzw+vp9cCUKYBGA3zyCZCTI+fW6Qz3c3IkFPbuLc/ZtEmCpvGxxtRzDhggZRg9Ov/vd8MGQ/i8dAlYuzb/Y/V64MwZeW/1euDff/M/NivLcN/DQ57j7CxljovL/3lmGD6JiIiqm/IIiUWp+dPrJay5uEgwAyTsJCYaatvULTNTQt4HHxjOOWsW4O0tIfL2bcOtev9//wOCg+W8r7wCrF5teEyvzyuGRr1VA93s2cCKFfl/b61bG8Lnn38CL79c+PuRmwv88480SecnJcX0eOOv8zvnmTOAr6/U2Lq6Am5ulrdhYYbn3HUX8Pzz8lhWFrBsmYRxlVYLNGok9yMigCNH5Pq4uEiwVO+7uACORrGxTRsJtoD8PNWta/IeF4Thk4iIqCyUR1PunfOWaVA0D4kvvQTcd5+hyVZtvs3KkiZY1ZYt0tRqfIx6/9YtabJVw4deD0yaBCxeLLViaqBUa/Nu3JAABQDPPScBszB6PfDGG6bBydytW4bwmZkJXL9e+HnVZuWWLSVsWdu8vQ3Ht2sHzJxp+vjt28D//Z9lqBs+HOjUCXBykuDm6Gi47+QENGtmOH7oUGlKVx+/fl3CoHGgU4NiaChw7lzh3xsgxy9davg6LEwCd26unO+99ww/V56e8vNbXKGh8seG+nNVCIZPIiKqfsoh0DlOnmzalFtYf7/cXNOmW7WPXXa2hJhmzUyDokYDPP440LGjac2fVgssWWI474IFwMGDhseNj1UUIDnZNCQ++6xs1jz4oKGGcssW2YojvyZc4+bbGjUAf3/TGjYXFznmxAnT5ykKMGyYBCp3d6nlM75VgycAPPmklN/48aQkoEkTy0A3ezbw3/8W7Xvq3dvQVG6sSRPLUNe9e9HOCUjANQ65aqDLLyiW1MSJ0nR/5owhyJaFiROBrl2B5s0LPVSjKAX9CVE5pKSkwMfHB4mJifD397d1cagAOp0OkZGRGDx4MJycnGxdHCoAr5X9sJtrVR5NuaU5b26ujMhNTZUAVbu27F+3DnjiCQkyGo30nevQwdAPLyhIQp5qyRJpFjbuq6fehoXJAAxrTY5Nm8prqIEyLAzYv9/weJs2UpNoTUiINO8WpSnTx0dq/FT9+wM7d1o/VqOxXnNYo4YEH+OmWzc34KefpGZPfd9iYiyPcXWVcDt3rmlZHRyAH34A6tSxDJaenvJ4Qaw15Wq1wIULpfv5WrcOypQp0OTmQtFqoXnvvdIPDFJdvlz2oa48zllO1LyWnJwMb+MgbYY1n0RE1VF5N+UWVPOXm2uo5TO/dXcH6tWT4xRFaqTeessQFEeMkBrB1FSpaZo2zXDeDh1k4EdqqoTOjAzDY337Aj//LN/35MmGAKYoMtBj0ybDsR07mobP9esNfdvMtWghTdbWAqJ5rZ2j2a9cNdSpnJwMzbg+PnJ9rJ23WzcJdGptnvkv+XnzgMceMzxuXPN36xZw992Wge7o0cJ/DgoLaN7elrV0AwcW/JyClGPNX06fPvhz40Z0GjMGTmXZRSI0tOwDYnmc08YYPomIyko51vyVaV9C86D4+uvSN83DAwgMlGMyMoDt200Hcxjfj4iQJk0AiI2V/n0qtb/fsmVyf9gwYM0aeSwzU4JQfoYPB776Su5fuQK8+abhMUUBtm41fN2vn2n4PH1ampTNOToamo7zC3R9+0rNqJMT0KCB6WMzZ0qgNe6rp/bfq1lTrreDg2Wt36efyjnVgRvu7qbn/flnOc7ZWc5pXhN4+bLlebVaGVVd0M9Xv375PwaUT6ADyqc5t7yaiENDkdSqVZULdfaC4ZOIqidb1vypFEWO1+kkBKhN6tnZ0lcuJwfYtAmOzz1n6Es4a5b0ZWvSRI5NSAA+/NDQt898GznSUI5z52TwQ2KioQx6vTSXzp0r537jDdl/65bUMubnsccM4TO/gQ/q1CvGgz7Ma/vUfWpA8/Q07D992vp5779fmrTVqWRUX38t5/DyMmyenqYjq60FRa0W+Oij/H8O5s2zvt/Y++9bNuU+/HDBz/HxKfhxe+vzB7Dmj4qE4ZOIKreKnhJGrzf0E1S3lBTD/c6dpSyA9NdbtUqC4o8/Gs6v1vylpkqzMSB9/e69VwKlcZ9B1fLlwMKFcv/oUalZvMNkWpjVq+WLVavkNilJRhTnxzigOTmZBk9jrq6mNW+enkCXLoYmW+OBHW5uEmJVLVta9iV0cJBayrAwGUxivD8x0RA2nZwMwdBcfkHxrbes/yxYGwRizt6acsux5o+BjsraoUNFO47hk4gqL/OQ+M478jUgoev8ecvVPdT799wDNGwoxx4+LDVbmZkSfNRmXcByhY9vvpHawvy8954hfCYkyHnzY1xzpyjSHzE/xkFU7fvn4CBlNmccxgICJKCoffvMt1atDMcGBQE7dkg/PPNAd/q0aRjx9pZl84qibl2ZKsc80N1/v/Xjizpw1M5q/qL/DcO0L6bg/Xt80bkMuxEevBaK+ctC8frrQATzYpmIjtbguee6IjBQg86dbV2agh08CMyfL71jjP4mrXTnBEzn2S+QYgeSk5MVAEpiYqKti0KFyM7OVr755hslOzvb1kWhQmSfO6fsefFFJfvcubI76aVLivLrr3JrTU6OoiQmKsrJk4qyf7+i/PCDoly/bnj8558V5eGHFaV/f0Vp3VpRJLIZNgcHw7k3bLB83Hj7/HPDebdsKfhYQFGiouTYHTvka61WUWrUUJQ6dRSlRQtF6dxZUfr1U5RvvzWcNy5OUV55RVFefFFRNBrLsv71l+HY9HRFiY1VlDNnFOXCBUW5fFlR/v1XUZKSFCU5WVGysqy/nw4OygGEK73xi3IA4VKu/N7fovrwQzmP+n1++GHpznfHge+vKb3b3VAOfH+tTM6X59IluT6l/b6NHDigKL17y21ZmT49RwEUZcaMnLI7qaIoM2fKpZo1q0xPazfs6VqVR1nL4/qX5TkvXFCUPXsUJTJSUXx9Ja8lJycX+BzWfBJVR+++C8fp003nJLzvPhngoTYLG286nTTB+vnJ82Njgb/+Mn18zx7giy8kehk3ZW/cCLzwgkwqfeuW5TQvP/0ktVAAcPGi6ahjAAcRjvl4Ha9jPiL00YZl4Hx9pUnXfHUP9X5QkOEkzZvLPIbqCh+vvJL/Ch+9eklfSVfX/JuDVWFhwKJFcj84GAcnv4/5+uV43WERIt6fLKOvVe7uln0UC3On5u9/T2QgSumDjzXjEPHek5W25u/j7YGIOgx8sgOIuLdMTinKoYn444+BqChZ+bA0NT8XL0plukYDfPGFdFvYvNkBjz0mP2IBAVIxrCiWMzWZz9pkvC8uTj4yOTlSRkA+GuPHm563simPGrWyulZnzwLx8fIef/65XKvPP3fA2LHyX0BZvKelKWturoyXu3kTOH5cJlhITTU0rmzYIOMBXVxksaV69Qy9YMx7xVjrzWL8s7p5M+58/6Y/U0FB0qh044Zs6v2Cbq9eLf77xHk+qUzZzXyE5cUW6xpnZsr/AImJctu5s2FU7VdfAV9+aXg8MVEGf9y+bXoOrVYGnLz2Wv7l+P13w4TJb74JPPVUweVW5+PbudN02hpABoL4+0uY/b//A/r0kf3Hj8sIa39/aRaeOBGz9KvwFmZhFt7AG9pnSj/HHwCsW2cZFMtgnr9Zj6firQ1emPV4Kt5Y51Wqcxn/ohjYPxfXk7So6Z+Ln3ZoK1X4MC7noEHSE6FWLekCW5nKCVgvq7+/jNe6fVt+qfv4yN8e6en5j+Eyfuz7741fQYH00FVvhVZrWESnLH39tfSqqF+/8CkzzZVXs+usWdIl13jsWlEYB68bN2SWKjV8vfWWvOdubvI3cna2PMfJybAKp/FtfveLcg0aNJDuzsZj1szHr5nvS0uTQOvhAYwbJz9j6n9t6mqZWq387X3zpuHW+P6tW4WvrFkcDg6mYdTNDTh5suzOn78UAJznk6jiFGe0s7paifqnaXy8/Plo/ltuxw75E9q4NjEnR/rXqWEzLc303H//bejnd+yYRU2iVbm58j+0OtWMOo2M8WY8TUy9etJvUJ125sYN4LffLM955oz8lt+92xA2/fwMo7rNNW8ONG9uCAmLa2Dzi10AAJ9jNMY/WxfKv6EIyC1loJk4ER/vfQhRG7zwyYSfETGx5EFRLSsAfP69nOezbV4YECm/8Dw85Fsv7Bej+X3TxVYkWVxPckB4uGGvTmc5dWRRlSZ8ZGfL30QXLxr+bjCWkACTclaWKg51+lBjSUkyu1PZ0JjdivxCj0ZjOXuTk5NsRVkZUi23u7t8dFq1krFf6hYcnH/lfVnVJgJmn4E7ff4+/VTKoC6m5OBgCJbqrfH95OTCf05u3y7+AkvFVdQVKwtz40bJ/5718JAu3/l1EddopNHFxcUw+5m6qe+hOm7S/NdDUTg4GP6rNv5vW71v7TYuDujZs+ivwZpPKlN2VfNZklpKRZE/xdVaRDUAnjkDvPgiDirtDU3EmkPyGzg31zJU3r4t/8upI2Lnz5c/k60waXbWxshqKubLwGm18j9AQID8r9+unez/809g717ZHxAgx+h08r9EWa4acmclkoP6dqZlLeY5c3LkF5naAi6s1yYNHmwYLG3t1tq+1FQJdk5OMmg8JUVqLp54Qn7Za7XS2m5t/JJ63/zrkjQ5lRUHBwkYYWHyNhvfqveDg+X7MpdfDZW6+uLFi/ILJS7OcF+9jY8veqBs1gwYM0am0YyIKHlYLon0dGDXLvkbbscOyznfzdWsKc2a6lgtDw/rY7jM98fHA888Y3m+b76Rj6JxqDQOmtaui7FDh0xDvGrWLAkm//wjjQXGK1Ua8/MzDaM1a8p/AT4+BddS63SGmjnjgJjfbVHHpRWFh4f0qNFo5L8Vaz9nDg6yKFXXrhLAXF1NbwvaFxsrc+yb+/57+d6NJ7mwtplPhJGaCly7Jr8G8tOihXwOfH1lISlfX9P7xvt8fAyzkeV3/aOjgfbtLferi2kZT8lrPk3vP/9Ynzns44/l/fT3l7GGxa1JV8uq0aRAUVjzSWSdeS3lokWyaoh5qExMlE+lWus3ebK00eXjY4xDFPrgE4xFhBIt1Uv5MV59JSBAkoLxb7TMTODQIdNz5kZL9cZ33xkCZUBA/v9bdOpkOiWOytqchKVpyr7TN/HjO30TPymgb2J2tgxSP3PGcrtwwXTQt7BemxQZWfLiGktNBVauLJtzWePlJb9QivKL0fz+jRvA2rWW5wwOlh9NnU666V65kv/ra7WGgOrnJ7/oAgOB//1PHl+3TgJ0fLzUtMXHy3tSGFdX+WVdp478uH77rfXjYmOB//xHNm9v6VLbt69MRtCsWeHdaotDr5fVH9WwuWePvEcqBwcJAkePWj43v1/oRaFOL+PgoECv1+TdhoXJ+1Na6mxT6u348Yay5uRIX8Z//jFsR4/K39U3bkhvmd9/z//c5rXUnp4lqy3Lj0YjayO0aiXhys+v4FvjaWDzC18HDpT8WqnrG5hfq+BgCeglVdygWBzm1z8/Go3h/xBfX+vH1Kpl/ZwtWhgmBymJWrWkv2hISNGmW2L4pErv4A//Yv5/nPH6S9mIuDfQ+kFZWfK/lvqb7I8/5H9gtUe0ut24IX+mnj1r+BTr9cDLL+dfgBUrDG286idabUsNCMBFj+ZI1NaCZtcubMYoAHeaiDWfQHl2MQKa+KFufQeL6hPF1w+K/s7Q6DnzoTw9P2+o9MWLQOLxf4HhI/A5RhvO6fAplIABCGgRWOpm57KakzCvibzdRGz2ywWSgE1+03G3txZxKwzNa2rAjIsr+D9QV1f5TzAgwLIlH5DpMENDJcTqdMW7PX1afnFZq03RaCQUtWljOX7J2ngm4/tnzgBDhlies7S/eA4dkvBp/ovy+++Btm1letHLl6VvnPntpUsSKnNyZN/ly9ZfIz1dugWbq1lTglOdOoaQaXwbEGD4uB06JOHT/BfaV1/Jx+2XX4Bff5Wasm3bZAMkFPfpI+973775B7WCughcuSLdinfskFvzaUzr1pWxVf37y2udPy8hoai/0ItC/cVbu7aCjh2P4K+/WuPKFU3eL/rSnjcsTJpw162T62p8XkdH4K67ZDOeISwzU2p61TD6zz/SEFJQDR1gGjx9fAoPi+rtv/9an1P/4MGKC19FUV7XqjzLWtD1rwznBOT/5AsX5OeuRo0iPKH0g+zLH6dash+lnr4nO1umn4mNVZS9exXlgw+UmXhTpoTAG4rSs6eiPPigovTtqyht2ypKWJiiuLtLZktJMZxn4sTCp9Yx3xo1kql0Ro9WlBkzFGXJEkVZs0ZRbtwwnDc1VVEyMhRFURS9XlEuXjQ/jd7s1jCbjYOD5Ww8xdtMz/nmmzLrz+HDMluPXl+8t3r/fp3SqlWCsn+/rljP0+lkhqF9+4o2g5G1zdNTLt8DDyjKggWK8sEHhpl0cnPldaKjDTMWGd9GRxfv+zSnntd8K815y6usly4pSlCQooSH5ypPPnlYCQ/PVYKCij7jUE6Ooly5oih//qkoX36pKI8+aiibtVmsFixQlBMnZFaokpSzQwdFefdduTUvZ06Oohw8qCivvSYfM1dXyzI0bqwoU6dKWZOSDM81nhYmPV1RfvpJUZ5+WmbAsvazNXSofHRPnbL8XBSlrCWRmakoWVky1VxWVraSmVm68xmfV/0e9Hql1Ofdtcv69X/rLfk5OX1aZkPLKeYMROXxGbCna1WeZS3L619e51Spea2wqZbY59MOlNeoxDK3bh0OPPEeFiiv4jXNQnRY8bB0LDKfs2H2bEOVyYsvSpWJ+tid4X4XUQeJCIBGo8Eg5QckIBC18C9+xCAo0CAAiaiLONPXP3/eMKLgnXekXdbf37RXtL+/9MEcM6ZY/RMVRR4+dEhqs6Kj5X5+i8XYgqen9Voq9X5IiGl/uxkzcvH221rMmJGLt96Szme3bxuacdXaMuP7ly9LDUdx/qLv3FmaWBs1Mmy1ahXe3Hr5ssxUZP4X+oEDpeshoDaPmddQlKaWsrzKCkilvkajw48/RmLQoMFQFCe4uJT8fOXVPGjc+KD2PSuonJmZsujTL7/I9tdflj9XTZsCHTvKfxHJyYZxasZN6RqNvPf9+8vWuXP+49lKWtaisoc+7+Xx8w+U32fAnq5VeZXVnqh5jX0+q4CyHJWoKlJTtjlFkXB49aqhk9jVq5JGZs8GJk/GJ8oqRKEPPlXGoMOc2cCcOSanuA1X3Bw8CTd0XtJZfbcfbka3xA344SZ882434ZE7r5n3DxJQC+EwdCZ5Z/o/qF3fGSGN3FC7qTdqhXkjr9fjk0/Klp/09Hz7JyqKjAVSA6Z6e+OG5WkcHaWfUJ06hqZEY5GR0t9JHdhenC0mxvrowWeekV8WxgNAEhKkuez4cdms0Wqlr5/a7PLbb/Juvf++A376Sfr7JSfn/5aZf9+1a8tbpg6Q37jR8rjS/EJTm3HU/8wnTy6b/8zLo9mpvMoKyDnUsKXRWF8avSTKsnkQMP1e1b5nBXF1ldUwe/cGXnpJfvZ++02C6M8/y8/xiROmg4SMQycgcxX27Vv0BZNKWtaqpLybXcv6M2BP18qeymprDJ+VVFEmgy1xn7916/DxpAxEYSY+ue9NRHzoAYwaZQiVV69KR62nnzZUT02dityPNyLttgNS4YUUeOdtZ9EA165mIF3/KtZD5nN8H5MRgzZI0fgiXeuFNI0XbuZ6I1PvApjMsz29CAW2PuDkybdNe4c7OkofspAQCUS1a5ver11b+r9lZpr2T9zoMx1BCVocHytdQY8ftx7CnJyk03x4uGzt28vXrq4STrdts/yFHhgor1sSnp5ya37ORx6xDHS3b8svEGsjky9elMd0OsPlNX4/s7M1OHPGcC53d8Oc3mrANL9fs6bp+KZDhyR82jrQFEV1/yVZXuGjtHx8gKFDZQOAt9+WUd3Wfo4cHWXi7YceqtAiVgnl/YeSqjJ/Bsj2GD4rKWvz0ZmPSpwxwzAFo1abz33kwvF2KhxTb+LW1XTcDgiD45uX8D88DQDYgAnApA1Im/Q5cqEFoLkTKlsj9bNcpKQ7IiUFSLm+Gmm6d/Mv8OcAYJi/IRNu+B29pNLSbPSyVlu0Duw3b0r+NTdtmgTwq1cNTcT//ivBUh1oUTTS1Jx0S4tnnzV9xMVFaizbtzeEzRYt8v/P1NYdw93cgCZNZLNGr5e/Jz74QHo6WJt3UKuVwS1PPFH8EciVNdDkpzr/kizP8FGWpk+XRbWsdRH488/SD2Kpzqrzzz9VDgyfZawk/TOvX5cmVuNN7TNSkDVrinJ2LYAadzbVkrx7qfDGm3jK+lOjjb9wzbvn6Ci1FN7esmVmAqdOWS+vVivvx4MPGkKll1fRwk3e9CUaBXpFk3c7caLlL56cHAmgahhVN+OAeuVKwVPIaDQSnKZPl6BZnG5A5fELvSzP6eAgtcAvvCCjsq39Qv/rr8rXRE7lw97CR1nXqBORbTF8lrGC+mfq9dKsax4085ukukYNWXLL3LQaG1Er4yJysnORCy1yZj6NHEc35OQAuVG/IeefE8iBozwGR+RonJDj6oULmvo4dLspFMUy+Wk0CoYM0aBrV0OoNN+8vOTWxcUyPOY3iKE0gcZQm6a5U5umybc2Te2DWFgTd2qqTMdiPCWJqrRTgpTHL/TyDAnm0/eUlr0FGqr87K1GnYiKhuGzDFjrn7lpk4SxU6ek393Zs8CRIzKnnjWN6+vQtvZ1tHU7hbaZf6DthqcQfzIFEfcGwgG50EObdzvx1gq0x2F5oo8PMOMBQ3vrbwCO5QKhwYY0VqtWXge9/ELiwYOaMmnGKstAUx61aV5ehi4N1bU2pbznuSMqK6xRJ6qaGD7LgLX+mdevy+Agc66uMkilbVA82qbtRdvk39Dq/DZ4nY8DzhsdeKMv9Gk6BEGPMFzCRKzDOkzEJYSh1nNTgTE9JViqo1JUPXsWaYFV86bs0iqvQFMetWnVvTZF/YWu0eTixx8vYvXqFlAUB/5Cp0qJNepEVQ/DZymcPSsjnJs1kyXkrNFAweCm5/CI309o+5/70KRfXZlrccVnwNy5hgO1Wlmiok0b2YKCEBqswQVNAzgrmdAAmIz3ke3gBpfJp0o8cVpxmrKLw54CDWtTym/6HiIiosIwfBaDXi99GNXl4Y4dK/w5BxGO9ifuNJHf8gMc78yP1KsX8NRThrDZvLlUi5px+WANMGUKkJsLjVYLl/feKtWMvfY4H2F5YG0KERGRbTB8FuL2bZn0+Ntvge++kxHVKq1WQY9OWRj2kCvqe13HsIk1LfpnIqQ20K2RBMy2bQ1PVufvKczEibIw8ZkzsjRMaZdLAYMXERER2U61DZ8FTYmUkAD88IMEzh07JICqvFyzMajmQQy9+TEGp22G74Cngaeex+UvTiMIOZb9M1c/CzzYpXSFVWf2JiIiIrJz1TZ8mk+JdOKENKV/+62sN2w8Z2WY5w0M1UZiaPLH6JW5C86X7rQte3nJYq4AQrvWsd4/s8upiv/miIiIiCopuwqfMTEa9O1b8ucbT4n0+We5ALT48AM9vvnGAXFxpse2b39nqbfBOWjbtz40ySkyKKhjR6BfP6B/f7mvzkQeGlrm/TOJiIiIqhq7Cp9ffFG08JmZKavZXLoEXL5sWHLxnXeMj5KlFTNumwbPtzENQ+r8jbDoPXf2OAL/97os0t27t8yrmZ9y6J9JREREVJXYVfjcutUBjz0mg34yMw1reRsHzMuXZY7N4nKEDh9hAsb4/gh07AtkZADu7vLglClFPxH7ZxIRERHly67CZ1IS0KWIY3dcXWUS8bAwyYLqbdY/pzH77cYWx/85bDnaL54NtP9YmteJiIiIqMzZVfgEDCvxBAbK5O7WAmZYGODnZ7n+OAAc+tEbeBuWUyJNmQJ0CKzA74WIiIio+rGz8CkOHizaFJkWdDrUahWIIO8MhKUcw0R8iHWYhEveLVCrFYMnERERUXmzq/Cp0ShQFOs1moXKzQWGDUNo48a4cOW/cL4ZDM3ZhzG5YTCya7lzonUiIiKiCmBX4bNNGwXXrpVwHfLFi4EffwR27YLL1Kl32uxDoQHA3ElERERUMewqfO7cmQsvrxIsB7lpE/Daa3J/wwYJnkRERERU4RxsXYDiKNE65IcOyfybALBwITBqVJmXi4iIiIiKxq7CZ7ElJAD33y+Lsw8aBLz0kq1LRERERFStlSh8rl27FvXr14erqyvCw8Oxe/fuAo/fuHEj2rRpA3d3dwQHB+Oxxx5DUlJSiQpcZIoCPPSQzDzfpAnw2Wecv5OIiIjIxoodPjdv3ozZs2dj8eLFOHz4MLp3745BgwYhznxx9Dv27NmDcePGYeLEiTh27Bi2bNmCAwcOYNKkSaUufIE0GuCpp4DatYFvvwVq1Cjf1yMiIiKiQhU7fK5cuRITJ07EpEmT0KxZM6xevRphYWF4x3Th9Dx//PEH6tWrh1mzZqF+/fq4++67MWXKFBw8eLDUhS/U8OGyznrTpuX/WkRERERUqGKNds/OzkZ0dDQWLlxosr9///7Yt2+f1ed07doVixcvRmRkJAYNGoSEhAR8+eWXuPfee/N9naysLGRlZeV9nZKSAgDQ6XTQ6XQFllETHQ2lZk2gTh3ZodUChTyHyo56fQq7TmR7vFb2g9fKPvA62Q9eq/JR1PezWOEzMTERubm5CAw0XQ0oMDAQ165ds/qcrl27YuPGjRg1ahQyMzORk5ODoUOH4q233sr3dZYvX46lS5da7I+KioK7u3u+z3NNTETPuXOhURTsXboUqfXqFe0bozK3c+dOWxeBiojXyn7wWtkHXif7wWtVtjIyMop0XInm+dSYLTGkKIrFPtXx48cxa9YsPP/88xgwYADi4+Mxb948TJ06FevWrbP6nEWLFmHOnDl5X6ekpCAsLAy9e/eGv7+/9ULdvg1tnz5wuHULSuvW6D5+PODhUZJvj0pBp9Nh586d6NevH5ycnGxdHCoAr5X94LWyD7xO9oPXqnyoLdWFKVb4DAgIgFartajlTEhIsKgNVS1fvhzdunXDvHnzAACtW7eGh4cHunfvjpdeegnBwcEWz3FxcYGLlQk9nZycrP+QKAowfToQHQ34+0Pz7bdw4gAjm8r3WlGlw2tlP3it7AOvk/3gtSpbRX0vizXgyNnZGeHh4RbV1Dt37kTXrl2tPicjIwMODqYvo70z5ZGiKMV5+fytWgV8+qn079yyBWBzOxEREVGlVOzR7nPmzMGHH36I9evXIzY2Fk8//TTi4uIwdepUANJkPm7cuLzjhwwZgq+++grvvPMOzp07h71792LWrFno2LEjQkJCSv8d7NgB3KlVxapVQO/epT8nEREREZWLYvf5HDVqFJKSkrBs2TLEx8ejZcuWiIyMRN26dQEA8fHxJnN+TpgwAampqVizZg2eeeYZ1KhRA3369MFr6lrrpaEosma7Xg889hgwY0bpz0lERERE5aZEA46mTZuGadOmWX3so48+stg3c+ZMzJw5syQvVTCNBvjuO+D114FFi+RrIiIiIqq0ShQ+KxV3d2DJEluXgoiIiIiKoERru9vcsmXASy9JszsRERER2Q37q/n8+mvghRfkfpcuQN++ti0PERERERWZfdV8Hj8OjB0r9596isGTiIiIyM7YVfh0HDcOSE8H+vQB/vtfWxeHiIiIiIrJrsKn5sIFmUB+82bA0f56DBARERFVd3YVPhUAGD8eCAiwdVGIiIiIqATsKnxqABnlfvmyrYtCRERERCVgV+ETAJCbC5w5Y+tSEBEREVEJ2F/41GqBRo1sXQoiIiIiKgG7Cp+KgwPw3ntAaKiti0JEREREJWBXQ8ZzDh8GWre2dTGIiIiIqITsquYTtWvbugREREREVAr2FT6JiIiIyK4xfBIRERFRhWH4JCIiIqIKw/BJRERERBWG4ZOIiIiIKgzDJxERERFVGIZPIiIiIqowDJ9EREREVGEYPomIiIiowthX+FQUW5eAiIiIiErBvsJnXJytS0BEREREpWBX4VPzxx+2LgIRERERlYJdhU8Hhk8iIiIiu2ZX4VOzf7+ti0BEREREpWBf4fPMGeD6dVsXg4iIiIhKyK7CJwBg715bl4CIiIiISsj+wmd0tK1LQEREREQl5GjrAhSH7sABIDzc1sUgIiIiohKyr5rP+vUBjcbWpSAiIiKiErKv8ElEREREds2uwqfm55+BkSOBN96wdVGIiIiIqATsK3zGxQFffQV8952ti0JEREREJWBX4VPfubPc2b8f0OlsWxgiIiIiKja7Cp9o2hTw9QUyMoCYGFuXhoiIiIiKyb7Cp4MD0K2b3N+927ZlISIiIqJis6/wCQDdu8stwycRERGR3bHf8LlnD6Aoti0LERERERWL/YXP8HDA2xto3Bi4ccPWpSEiIiKiYrCr5TUBAM7OwPXrcktEREREdsX+aj4BBk8iIiIiO2Wf4VOVlmbrEhARERFRMdhn+MzJATp0AHx8gKtXbV0aIiIiIioi+wyfjo5Abi6g13PKJSIiIiI7Yp/hEwDuvltu9+yxbTmIiIiIqMjsN3xysnkiIiIiu2O/4VOt+fz7b+DWLZsWhYiIiIiKxn7DZ3Aw0LChrHK0f7+tS0NERERERWC/4RNg0zsRERGRnbG/FY6M3XcfkJ0NdOpk65IQERERURHYd/gcOVI2IiIiIrIL9t3sTkRERER2xf7Dp14PHD8OxMTYuiREREREVAj7D59r1wItWgCLF9u6JERERERUCPsPn126yO3evbLkJhERERFVWvYfPtu0ATw9geRk4J9/bF0aIiIiIiqA/YdPR0ega1e5z3XeiYiIiCo1+w+fgGGpTU42T0RERFSpVY3wabzSkaLYtixERERElK+qET47dgScnICrV4ELF2xdGiIiIiLKh32vcKRydwdWrwbq1weCgmxdGiIiIiLKR9UInwAwbZqtS0BEREREhagaze5EREREZBeqVvj88Udg4UIgMdHWJSEiIiIiK0oUPteuXYv69evD1dUV4eHh2F3IFEdZWVlYvHgx6tatCxcXFzRs2BDr168vUYELNG8e8NprnHKJiIiIqJIqdvjcvHkzZs+ejcWLF+Pw4cPo3r07Bg0ahLi4uHyf89BDD+GXX37BunXrcPLkSWzatAlNmzYtVcGtUqdc4mTzRERERJVSsQccrVy5EhMnTsSkSZMAAKtXr8b27dvxzjvvYPny5RbH//TTT/jtt99w7tw5+Pn5AQDq1atXulLn5+67gXffZc0nERERUSVVrPCZnZ2N6OhoLFy40GR///79sW/fPqvP2bZtGyIiIvD666/jk08+gYeHB4YOHYoXX3wRbm5uVp+TlZWFrKysvK9TUlIAADqdDjqdLv8Cdu4MJwDKoUPIuXUL8PAozrdHZUC9PgVeJ6oUeK3sB6+VfeB1sh+8VuWjqO9nscJnYmIicnNzERgYaLI/MDAQ165ds/qcc+fOYc+ePXB1dcXXX3+NxMRETJs2DTdu3Mi33+fy5cuxdOlSi/1RUVFwd3cvsIz9AgLgnpiIv958E4lt2hTxO6OytnPnTlsXgYqI18p+8FrZB14n+8FrVbYyMjKKdFyJ5vnUaDQmXyuKYrFPpdfrodFosHHjRvj4+ACQpvsHHngAb7/9ttXaz0WLFmHOnDl5X6ekpCAsLAy9e/eGv79/gWXT3nMP8Pnn6JyTA/3gwcX91qiUdDoddu7ciX79+sHJycnWxaEC8FrZD14r+8DrZD94rcqH2lJdmGKFz4CAAGi1WotazoSEBIvaUFVwcDBq166dFzwBoFmzZlAUBZcvX0bjxo0tnuPi4gIXFxeL/U5OToX/kPToAXz+ObT//AMtf6BspkjXiioFXiv7wWtlH3id7AevVdkq6ntZrNHuzs7OCA8Pt6im3rlzJ7p27Wr1Od26dcPVq1eRlpaWt+/UqVNwcHBAaGhocV6+aB58EIiNBbZuLftzExEREVGpFHuqpTlz5uDDDz/E+vXrERsbi6effhpxcXGYOnUqAGkyHzduXN7xjzzyCPz9/fHYY4/h+PHj+P333zFv3jw8/vjj+Q44KpWAAKBpUyCfbgBEREREZDvF7vM5atQoJCUlYdmyZYiPj0fLli0RGRmJunXrAgDi4+NN5vz09PTEzp07MXPmTERERMDf3x8PPfQQXnrppbL7LoiIiIjILpRowNG0adMwbdo0q4999NFHFvuaNm1asSPKDh+WlY5q1JB5P4mIiIioUqhaa7urMjOBzZul36ei2Lo0RERERHRH1Qyf4eGAqyuQmAicPGnr0hARERHRHVUzfDo7A506yX0utUlERERUaVTN8AkA3bvL7Z49ti0HEREREeWpuuHz7rvlljWfRERERJVG1Q2fXboADg7A+fPAlSu2Lg0RERERoYRTLdkFb28gIgLQ64GEBKB2bVuXiIiIiKjaq7rhEwD27QO0WluXgoiIiIjuqLrN7gCDJxEREVElU7XDpyo9HcjKsnUpiIiIiKq9qh8+R4+WZTZ/+cXWJSEiIiKq9qp++HR3B3JyOOUSERERUSVQ9cMnJ5snIiIiqjSqfvhUJ5v/6y8gM9O2ZSEiIiKq5qp++GzUCAgMBLKzgYMHbV0aIiIiomqt6odPjYZLbRIRERFVElU/fALs90lERERUSVTtFY5UvXsDDz4IDBhg65IQERERVWvVI3y2bg188YWtS0FERERU7VWPZnciIiIiqhSqT/hUFODUKSAqytYlISIiIqq2qkezOyAj3Xv2BEJCgMuXZRQ8EREREVWo6lPzGREBODkBV68CFy7YujRERERE1VL1CZ/u7kB4uNznfJ9ERERENlF9widgmO+T4ZOIiIjIJqpn+ORk80REREQ2Ub3CZ9eucnviBHD9um3LQkRERFQN2VX4nD/fAdnZpTiBvz/QooXcZ+0nERERUYWzq6mW1q/X4uRJYMsWIDi4hCdZtgxwdJRpl4iIiIioQtlV+PTyUrB3rwxa37oV6NKlBCcZMaLMy0VERERERWNXze4//5yD5s2B+HipuHz3XVm4iIiIiIjsg12Fz4YNgT//BB54ANDpgCefBCZNAjIzi3miPXuA554DDh4sl3ISERERkXV2FT4BwNMT+OIL4NVXAQcHYP16oEcP4NKlYpzk3XeBl14Cvv++3MpJRERERJbsLnwCsiz7ggXATz8Bfn7AgQPSD/S334p4grvvlltONk9ERERUoewyfKr69ZOW87ZtZdrOvn2B1auL0A9UnWz+jz+k/Z6IiIiIKoRdh08AqF8f2LsXGDMGyM0Fnn4aePRRICOjgCc1ayZVphkZRUyrRERERFQW7D58AoC7O/DJJ5IjtVrgs8+Abt2A8+fzeYKDAzB9utyfPx+YNo01oEREREQVoEqET0D6gT71FPDLL0DNmkBMDBARAezYkc8Tli4FXn9dnvjxx8DZsxVZXCIiIqJqqcqET1XPnsChQ0DHjsCNG8CgQTIy3qJlXaMB5s0DvvlGqkqbNrVFcYmIiIiqlSoXPgEgNFRGvk+cCOj1wKJFwEMPAampVg4eOhQYNszw9Z9/FmPYPBEREREVR5UMnwDg6gp88IFM6enkBHz5JdC5M3DqVAFPunRJgmi/fjKBKBERERGVqSobPgFpWZ8yRSoyg4OB48eBDh0KmFs+IADo1UsGH02cKM3yubkVWWQiIiKiKq1Kh09Vly5AdLSMgE9JAYYMAZYskSZ5E25uwKZNwAsvyNf//S8wfHg+7fVEREREVFzVInwCUvP566+GGZaWLpUW9ps3zQ7UaCSZbtoEuLgA330nKyJdvFjRRSYiIiKqcqpN+AQAZ2dgzRrgo48kV37/PdC+vSzPaWH0aGmvDwwE/v5b1oInIiIiolKpVuFTNX48sG8f0KABcOGCNMe/+aaV6Zg6dQL++gsYO1ZmsCciIiKiUqmW4ROQGs9Dh4CRI2V80VNPAQ88ANy6ZXZgnToyCb2Hh3ytKMDWrVySk4iIiKgEqm34BAAfH2DLFqn1dHICvvpKQunBgwU86ZVXJKU+/DBw+3aFlZWIiIioKqjW4ROQ8UUzZwJ79wL16sl68N26Sd9Qq5WbwcGAoyOwebNMy3TtWgWXmIiIiMh+VfvwqerQQZrh778fyM6WQDpqFJCcbHbg448DO3cCfn7SH7RjR+DIEVsUmYiIiMjuMHwa8fWVpvdVq6Ryc8sWIDxcQqmJXr1kGc677pJVkbp1A7Zts0WRiYiIiOwKw6cZjQaYPRvYs0fGGp09K5PUv/OOWTN8o0bA/v3APfcA6emyePyVK7YqNhEREZFdYPjMR6dOwOHDwNCh0gw/bZqMMUpJMTrI1xeIjASefBJ4+22gdm2blZeIiIjIHjB8FsDPD/jmG2DFCsMYo4gIsy6eTk7A2rWyFrxqyxZg5Uouy0lERERkhuGzEBoNMGcO8PvvQFgYcPq01Iq+/34+o+H1euC554BnnpEnLFwIxMdXeLmJiIiIKiOGzyLq0kWa4e+9F8jKAqZMAR59FEhLMztQrwfmzpXBSMnJwGuvyRxOEycCsbG2KDoRERFRpcHwWQz+/jKo/fXXAa0W+OwzaYY/etToIEdHYNIk4PhxabPv1k06ja5fDzRvDixbZqviExEREdkcw2cxOTgA8+YBv/0m44tOnpSpPtetM2uGd3AAhg2TYfP79gHDh0sbfrduhmPS04Hc3Ar/HoiIiIhsheGzhLp1A2JigIEDgcxMqewcN85KMzwgbfZffQWcOQP06WPYv3Qp0LQp8O67XKqTiIiIqgWGz1IICAB++AFYvlya4T/9VFZK2rMnnyc0aCC1n4DUeG7ZIoH0ySeBunWlST4pqcLKT0RERFTRGD5LycFBBrRHRQEhIcCJE0D37rL98EM+I+IBSatHjwJvvCHB8/p14IUXZIT8zJmyyDwRERFRFcPwWUa6d5fR8E88ATg7S+3nffcBbdoAGzcCOTlWnuTpCcyaJbWfn30GtGsnze9r1sg8oURERERVDMNnGapVS+b/PH9eBiV5ekrl5qOPymqca9YAGRlWnujoKMsnRUcDP/8sHUnnzDE8fuQI8N13+SRYIiIiIvvB8FkOQkJkOqa4OODll4GaNYGLF6U1vW5d4KWXgJs3rTxRowH69gV+/BGoX9+wf8kSWeczNFQmr//774r6VoiIiIjKFMNnOfL1BZ59VoLn2rWSJxMTZQGkOnVkLvorVwo5iaIATZpIgv33X2mOb9NGmujfeEP6ihIRERHZCYbPCuDmJgPaT52Srp2tW8uUTCtWSCCdNEnmC7VKo5FVkq5ckRnuR4yQ9eRjYoDZs4HBgyvwOyEiIiIqnRKFz7Vr16J+/fpwdXVFeHg4du/eXaTn7d27F46Ojmjbtm1JXtbuqV07Y2KAyEigRw9Ap5MJ6ps1A0aOBA4cyOfJTk7AkCHA1q2yVvyaNTKv05gxhmNSUyWQRkcXMMyeiIiIyHaKHT43b96M2bNnY/HixTh8+DC6d++OQYMGIS4ursDnJScnY9y4cejbt2+JC1tVaDTAoEGyStLevdKdU1FkHvqOHaXb586dBeRHf39g+nTgr79ktLzqyy+lKT4iAmjVCvi//5OgSkRERFRJFDt8rly5EhMnTsSkSZPQrFkzrF69GmFhYXjnnXcKfN6UKVPwyCOPoEuXLiUubFXUtSvw7bfAsWPA+PFSO/rrr0D//pIht2wpZAVOB6NL2Lw5MHo04OoqJ5w/XwYpDRoEbN4sSzGVg9xcGeH/00/Ae+854PTpGuXyOkRERGT/HItzcHZ2NqKjo7Fw4UKT/f3798e+ffvyfd6GDRtw9uxZfPrpp3jppZcKfZ2srCxkZWXlfZ2SkgIA0Ol00Ol0xSmy3WjcGPjgAxmM9MYbDli3zgGHDmnw0ENAo0YKHn5Yj7p1FYSGAqGhCmrXBjw8zE7Svj3w8cdAcjI0X34Jh08+gcO+fcBPP0HZvh05Z87IJPYllJICnDqlwcmTwMmTGpw6Jdvp00BW1p2Vm6AF0BORkblYvDgH3bsreYs6UeWifpaq6meqKuG1sg+8TvaD16p8FPX9LFb4TExMRG5uLgIDA032BwYG4tq1a1afc/r0aSxcuBC7d++Go2PRXm758uVYunSpxf6oqCi4u7sXp8h2qW9foEMHZ/zwQ31ERjbAmTPOePFFrcVxnp7Z8Pe/DX//TPj730ZAwG0EBGTe2dcYATOfRcDYSwiLioJbYiIOHz0qE48CaP3uu8j09UV8p05IrVs3b9nP3Fzg+nV3XLniabZ54eZN13zL7OiYi5CQdPj6ZuKffwIQFaVFVBTQrFkSHnjgFNq3T2AIraR27txp6yJQEfFa2QdeJ/vBa1W2MqxOZm6pWOFTpTFLEYqiWOwDgNzcXDzyyCNYunQpmjRpUuTzL1q0CHOMJllPSUlBWFgYevfuDX9//5IU2S6NHi2j4j/5JBeHDmlw5Qpw6ZIGly8D6ekapKU5Iy3NGRcv+uR7Dh8fBbVrT5Ha0m+B2rUVhHregsv2d+GtXMCVTVrEemtx0r8rTuob4cw1L6NaTEtBQQqaNFFw110KmjTBnVsFdesCWq0bdDpHfPrpzzh4sDc+/tgRsbH+ePHFLmjXTsGCBbm4/37FpKcA2Y5Op8POnTvRr18/ODk52bo4VABeK/vA62Q/eK3Kh9pSXZhihc+AgABotVqLWs6EhASL2lAASE1NxcGDB3H48GHMmDEDAKDX66EoChwdHbFjxw706dPH4nkuLi5wcXGx2O/k5FTtfkh8fU3HFAEyECklBbh82bBdumT69eXLQHIykJysQXIycPy4caD0B/Cj4cuUO9sdLo45aNzUEXfdBTRtCtx1l2Hz8dEAKLgKMzDwNt5+G1i6VIMVK4B33wUOH9Zg9GhHNGsmc5+OHi39W8n2quPnyl7xWtkHXif7wWtVtor6Xhbr17+zszPCw8Oxc+dODB8+PG//zp07MWzYMIvjvb29cfROM69q7dq1+PXXX/Hll1+ivvEqPlRkGg3g4yNbixb5H5eaahlIjbdbt4AG9XJxl/tl3JX8F5qe2oa7kvai7tuLoZ08UU5y4YJ0Rm14L+DZCdKns2iCg4H//hdYtEgG4b/5JhAbC4wdC7zwArBggQyysvJ3BhEREVVRxa57mjNnDsaOHYuIiAh06dIF77//PuLi4jB16lQA0mR+5coVfPzxx3BwcEDLli1Nnl+rVi24urpa7Key5+Ul84c2a1bQUVoAdWVTHpA+oaGhhoe3bQNeeUU2f38ZOX/ffcCAAUCNGkUqh78/sGyZrAy6dq0s0nTuHDBliuyfNw944gmgGnTnJSIiqvaK3ftu1KhRWL16NZYtW4a2bdvi999/R2RkJOrWrQsAiI+PL3TOT6qkNBpZfsnPz7CvRQtg1CipZk1KAj79VNrMAwKAXr2AM2eKfHofH6kFvXABWLUKCAmRhZtmzwbq1QOWL5fuBERERFR1lWjox7Rp03DhwgVkZWUhOjoaPXr0yHvso48+wq5du/J97pIlSxATE1OSlyVb6NsX+PxzWUN+1y6ppmzWTIbF798PBAUZjo2MhGbnTjgYTZNljYeHBM5z54D33pMlRq9fl76gdeoAzz8vOZeIiIiqHg75oKJxcgJ69pTt9dclOR45Anh6Go75z3/gePgw7tVqZc7Rbt1k69pVqjnNuLgAkycDjz8u+faVV6RP6IsvStP81KnSVB8cXPJiK4rMrZ+cLLWqKSmG+zodcM89phW9REREVL4YPqlkGjSQTZWbC0REQElIgMOVK7JI/YEDwOrV8vjddwO7dxuOV5S8uUUdHYFHHwUeeQT4+mvg5ZeBw4eBFStkCfuJE+XxrCzLAGktVJrvy8nJ/9twdwcee0xqYhs1KvN3iYiIiMwwfFLZ0GqB999HTnY2oj76CH1cXeH411+yeP3ff5sOYtLrpZNno0ZSK9q1K9C5Mxz8/DByJDBihCzV+dJLwL59Mkhp7drSFU+jAby9DZuPD3DzptS0vv22nH/YMKlp7dYNnBCfiIionDB8UtnSaHA7MBDK4MHAuHGyLzXVdCTRyZMyMemlS0BUlGF/s2ZA167QPPAABg0aiIEDgd9+A157TfKrGhqNA2RR93l4wGJye0WRl1+xAoiMBL75RrYOHSSEjhzJuUiJiIjKGn+1Uvnz8pJNddddwD//SK3ovn2ynT4t1ZCxsTKIaeBAaDRArzY30avHu8CCLkBEhGkf01LSaIA+fWQ7flx6CHz8sfQWGD1aBj899ZQ0+/vkv4iUXbp9Wy7B7dv8L4CIiCoWf/NQxXNwkCmcWrSQEUeADHffv1+C6L33Go7dt0+GwavPa94c6NQJ6NhRtpYty6R6snlz4P33pan/nXekKT4uTmpAlyyReUhnzQLuzChmd65eNeT8ffuAQ4cAnc4JTk4D8cknGgwfDgwdCtSqZeuSEhFRVcdVtqlyqFlT0s+rrwJduhj216gBPPAAEBYmfUX/+QdYt05mqG/XDvjkE8OxiYkyCl9RSlyMWrVk9aW4OFnYqVkz6TWwciXQsKHUiP71V8m/zYqg0wHR0cBbbwEPPyzda2vXBh58UOZX/fNPOcbTU4FOp0VkpAOeeEIqnLt3l24I587Z+rsgIqKqijWfVLmp0zUBQHy8tIn/+ackwAMHpBZU9dln0k4eEGCoGe3USTpx+vsX62VdXYFJk2QaqO3bJZD98guwebNsd98NzJkjeVlb9BVHy0VSEvDHH4Zazb/+AjIyTI9xcJD1A9TxXV27ArVr5+D993fjxo2e+O47LQ4eBPbskW3uXKBVK2D4cOD++4G2bTkIi4iIygbDJ9mP4GBJe0OHytd6vWkiun5d5iNNTJQRRJGRhscaNgR++EH6mxaDg4OsKDpokExrunIlsGmTIaQ1bCjTND32mAxqKm96PXDihIRMtZfCiROWx9WoIRXIatDs0MG02y0gtZ9hYamYMkWP55/X4tIl4NtvZdDVrl2y0urRo7IEat26EkLvv1+CNwdiUUVJSpJ5gHv2lF42RGT/+CuE7Jf58PUXXwT+8x8gJkaq/9Tt1Cng/HnT6Z7mzQN+/VUmw2/fXprwW7cucIH5Nm2A//1PlgFdswZ4913g7Flg5kxZlWnKFLlvZT59ANIbQKeT+UrNt8xM6/vV7coVCZv79wO3blme+667TGs1mza1fHsKExYGzJgh240bktW//lqmvbp4EXjjDdn8/YEhQySI9utX4FtGVGKpqdJN5L//lfuOjsDChfIRd3GxdemIqDQYPqlqcXGRpnbj5vibN2U4u3HV5N69Murm0CHDPgcHSW3t2gHr1wPOzlZfIiREVmNavBj46CP5BXn2rHRXXbECaNLEerjMzi5Vd9Q8bm7y7XXtKrWbnTtLT4Oy5OcHjB0rW0YGsHOn1Ih+953URH30kWzu7sCAARJE77uPq0VR6WVmyqC/V16RRgxAGj3i42VA4JdfSrfvrl1tW04iKjmGT6r6fH0N/UZVn38utaKHD0sAPXwY+PdfCanJyabBc9IkCbDt2hlqSYOD4eEBTJ8uy4B+9500ye/eDRw7VrRiabWSlV1d5bagzdfXEDhbt5beBRXF3V0m4B82TFaL2rNHgujXX8vArK+/lk2rlabR4cNljtTSLItK1Y9OJ3/QLFsGXL4s+5o0kQaNBx6Qn7Hp06Wbyd13Sw39K6+U6exrRFRBGD6peqpTR7YHHjDsi4+XIJqaatinKJIsExKAr74y7A8MlCDavTu0ixbl9Yf85x/JsGpoLChY2nqgUkk4OgK9esm2apX0cPj6awmjR49KT4Zff5Vpqbp3Bx56SIJoUJBty02Vl14vg/iefx44c0b2hYXJrBPjxxv6F48cCfTuLdOfffSRzOawbZtMkda/v82KT0QlwPBJpAoONp1jVPX556Y1pCdOSML88UcgPR1YtCjv0JbLx6BlzZpSPdmmDdCoubSTV0EajVQCt2sntVVnz0oI/fJLGX3/+++yzZwJ9OghQXTECAZREooi/YoXL5YVzACZcW3xYuk/7epq+Rw/P2DDBplCbPJk6Ys8YAAwYYJ0eWG3DyL7wPBJVBCNRqpbevc27MvIkN+Whw6ZTuGUnCzTPRlzcJC2w9atgcGDpSqnimrYUGqlnnlGmuO//BL44guZGeu332SbMUOa5h98UGqyAgMrtoyKImU7dEhmL8jNlWZbLy/DbX73K7KrQ1W3a5esHbF/v3zt7Q3Mny8zpRWlGb1/f2llWLxYakA/+kj+Fnz7bfm5IqLKjeGTqLjc3WWUT+fOpvu1Whmo9Pffsh05IqNzTpyQzcvLED6zsqTKpmVLCaatW8v9KtKBrU4dmQd1zhypnVKD6F9/SfDYtcuyRrSsg6heL824aqW1ut24UbLzOTtbBlJrIdXfH2jQQMJ4w4aV55LeuCETP5w6JavZnj4tY/A6dJCtVat8x9iVmQMHJDDu3Clfu7lJF43584tfa+npKbMvjBolS+CeOCG9aIYPlxDKPsf5O3BAZhE4d06C/MiR0oLBuXypojB8EpUVT0+Z8FOlKNKPVA2jbdsaHouNNVQHqjQaSSutWwNjxkgiqwLq1jXUiF64IEF0yxbTIKrWiKpBtLjLfObkSPgwDpkxMabdd1WOjpLz27WTvyNSU4G0NOu3qakySwEgt0lJshVHrVqGIGq+1apVtr/w09IkcBuHTPV+fqF7/Xq5dXaWH1E1jHboIFN4lUXf5OPHgeeeM3SbdnSUZvP//Kf0IbFrV7nWL78s06B9/TUQFSXN8I89xkClUhTpj718uSyYoTp4UAZu1a8vn72RI2VwY3GnaiMqDoZPovKi0ci8TCEhwMCBpo+FhcnSoMa1pNeuSXI4c0Z+86vOnAEefVQWoG/WzLDVq2d3o5bq1ZPVk+bONQTRL76QmpioKNmmT5cBTQ8+aD2IZmVJk6saMg8flrcvM9Py9Vxdpeut8XSuLVsWb57I7GwJdcaB1FpIVe8nJEj/17NnJagmJMimNjEb8/SUWtJGjSyDaViY9cn8s7Kkxso8XJ4+DVy9WvD3EhoKNG4sPUEaN5ZAeuCABJCbNw1T4xqXLzwciIgwBNL69Yse6M6fB5YsAT791LAmxNixMpioQYOinaMoXFyk3/EDD8iqZNHRUhu6aZMMSKpfv+xey97o9dIX+9VX5VoD8nM1Zox8zr7/XrosnD8vgX3FClmOV521ont3u/tvhuyARlHKYubB8pWSkgIfHx8kJibCv5jLJFLF0ul0iIyMxODBg+HETnLFk5AgQ8b//hvo00dSEyAJ7cEHLY93cZGqqeefN3R0y86W3zbWRmuYqUzX6vx5Q42o+gsSkNqXXr2Avn0lzB06JMEzJ8fyHF5eUnOnBs327WXaVluuxnTrlgRFNYyq25kzMp1QQf/7OjpKWG/YEAgO1uPIkeu4dasWLl7UQK/P/3kBARIu1YCp3m/YMP9VuBRFynXwoLz/Bw5IgDNfphWQbgXGYbRDB8vaS3VOzg8+kCmUAPlDYtkyoEWLgt6x0svJAVavlprWzEyp3X7lFaldL+8QVZk+U9nZwMaNwGuvASdPyj43N5k57plnpEVClZEhi0ls3SqTexi3GNSsKTN5jBwp/y1Vlf/WK9O1qkrUvJacnAxvb+98j2P4pDLFD3Q5uHZNho3Hxhq2kyelCgyQxKZOGfX99zIhZ/36prWk6ubjk3faynqt1CD6xRcShqzx8zMNme3bS7iyp6bCrCz5Xs2D6dmzsl+9vNZ4eZkGS+PaTF/fsilfbq78qKlh9MABqWFWw6Sx2rUNQfTWLVkB7PZteaxfP2kSN67MrwhnzkjQUnu2dO4sk9M3b15+r1kZPlNpacCHH0oNpjpfao0aEr5nzZIwWZCsLODnnyWIfvutaXeNGjVkdeORI6WvaBH+xq20KsO1qooYPskm+IGuILm5MpInNlZ+q6tt06tWySif/BgFVd2ZMziwcSM6PPwwnBo2rJRta+fOSRCNjpZwpQbNOnWqdl8+vV6WVFXDaFxcLpKS/sbIka3QrJkjAgNt8/1nZUnFvHEgPX7ceg1uly4SOo0niqhoer0EsblzpTbPyUn6mS5cWD6Dq2z5/19SkoT+N980BMbgYPnvYMoU+YOluHQ6Ce9bt0pf2n//NTzm6Skz040cCQwaVHkG1hUVf1eVD4ZPsgl+oG1MUeQ3hHEtqbpdvSrt1u3aAQByV6yAdu5ceZ6Tk3TCa9xYOiA2aiTtpBwyXClU5s9VWpr8WKlhNCUFePJJWW61svyBcPmylOn77+Xrli1loFVZ18ba4jpdviyrq73/vkw7DMjHd/58YNy44vVvLkhuLrBvnwTRr74CLl0yPObqKt3aR44EhgwxaWCptCrzZ8qeFTV8csARUVWi0cgs7kFBllVOycmmnf6cnZEaGgrPhARosrOlKV/tHAYAHTsawudnn8nWuLEhoDZuLFWQlbDGlCqOp6dMmdWjh61Lkr/QUFkNafNmmeLrn3+kGX70aPlRDg6Wj0xwsGyBgWUX2srLyZPA66/LuEW1K0S7dlKrO3Jk2X8stVoZfNS9uzSwHDggQXTrVsMCE998I8cFBkrXmKJuvr5SM1tZ/lipCvR64Pp1qXOIjzfd1H23bkmLkrpYSLt2FdeqxPBJVF2YVUfop07Fr3XqYPCAAXD691/pJHf6tOG2cWPDwX/8IcvRmFNrTL/9VgY/ATKLe2amjGio7L/BqdrQaCRs3nMPMHu2DMYxXxPCmJ+faSBV75vv8/Gp2NB08KCMXP/qK0N3h549ZaG1/v0rpiwajfxt2rGjlOXvvw1B9PhxCTeFzbxgztFRQmh+AdXbWwaPubnJbWH3nZxK/l4oitTop6SYbsnJhe9LTZUuHcZzAJtvBT3m5SXfQ35lz8mRxi1rYdL4/r//Sm11YU6elEFmKl9f0zDarp0E1LIeuMlmdypTbMqwH8W6VocPy1JFxgH17FnDqJjERMNqT089JR3PNBoZiVK/vuk2YkTJOqBVY/xclb2oKBnHFx8vY/rUX97XrlkfVJUfV1dDIA0M1CMz8yKaNasDDw8t3NxgdVMDUn6b+S96dY7OV1+VwUCqoUOlprNLl7J5T8rClSsycceNG0XbkpIKHlxXUlptwQHVxUWPxMR4eHkFIyXFwSJU2jIZaTSWATUnR34+ExKKXjaNRoYDhIQY/mAKDjZ87ekpfywcPizbsWPWZxJxc5NFKIwDaatW1leOZrM7EZUd9X8cY7m50uHs7FnT5Wn0emneT0+Xxy9fBnbvNjw+eLAhfL72mqSA+vWlBlUNqA0alN2wbSIrzFfNVSmKhCI1kJoHU+Pb5GSp5L9wQTbAAUB9/PRT6crm6GgaRtVlYQEJVY88AixYUP7TVpVE7dqyFcft24WH1NRUOS4jQ7b87qvTkOXmGubgtc4BQMEF1WqlZtvbWzbj+/nt8/KSaa7U1zafD9h8M39MUWQrqOxarWkNfH7hslatwmss77nHcD8rSwLo4cOycIN6m55uOQewVitT2RkH0rZti97dg+GTiEpGq5WmdeMJAwFZbPvNN6U29Nw5mTfo/Hm5f/WqTESp2rcP2L7d+vl9fOR5agjdu1eqJOrXlwkw7XmeF6q0NBqpxPf3Lzzc3b4tQVQNo5cv52LfvtOoXbsxsrO1ecGosC0jw7T2LyfHMny4usrUUXPnWn7k7J2bW8lCqzlFkVrrwgLq7dtAamouYmKOoVOn5vDzc7QIkT4+8p5XZJcKRZHyWQumDg6GcFmzZvlMK+fiYphRRGW8TLHxdv26BNVjx2QRCVWdOkV7LYZPIip7Go38D1mzpqzVl5/Fi6Xt0Dignj8vHZZ0OplYUPV//yd9S1UhIaY1pYsXl//i5ERG3NwMP4IAoNPpUbfuSQwe3BBOTsUb8aPXSy2qtXCamSlB2PjvNrKk0ch/Ac7Opv91WKPT6REZeR6DBzerNBPnazTSaOThITWblYGDg2E+4VGjZJ+iSD2CeSC9cMFQQ18Yhk8ish111IK59HSpSjKudmjQQNa9P3dOqgTUUQ1790onriVLDMeOHi2Tg5o35av3/fw4tJYqFQcHQ39EospM7c5fu7ZMqaa6eVP+Ox4ypPBzMHwSUeXj4SFz4BhbuVJuFUVGKRjXlN6+bRomT5yQtqIzZyzP7epqum7k2rVyvtq1pTZV/V/V15cBlYioiHx9iz7lGsMnEdkXjUbaHwMC8p8l/LvvDOtUmvc7Na/1/PBDaTMy5+oq00fFxBj2ffONdM5Tg2pICPueEhEVE8MnEVU9YWGy9epl+Zj5PDqPPCI97K9ckWb8K1ekJjQzUzZjy5ZZBlV/fwmhd90ly5eqTpyQuUxCQuxr0XkionLG8ElE1Yv56AJ1iVFjmZnS5zQtzXR/584SKNWgmpkpQTUpSeZXMTZmjKw76ewso/ON+53edZcMtCIiqoYYPomIzLm6GoYwG1u71nBfUaSH/ZUrsqkTDBrTaiWUnjolm8o8fI4dK/1WjQNqgwZcJYqIqiSGTyKiktBoDGv/tWpl+Xh0tEzYePmyZb9T83lUIiNlJm1rr9G9u+myNmvWSDN+rVqGrWZN6e3P5n0isgMMn0RE5cXRUZrc69XL/xhFAf73P0NANb5NT7ece+eFF6wHVa1W+rgaB9VXXpFb86Baq5Z0H+BofiKyAYZPIiJb0mhMJ8tTKYqsEmU8LZSiAA89JJPwJyTIMiMJCcCtW7KeoHnN53//K10DrOnYEfjzT8PXK1dK/9TQUBmsFRpafkupEFG1xvBJRFQZqatEAYYR+hoN8M47lsdmZUlQzckx7FMUYMoUCapqSFUDa3q65RIwL74oIdaYs7NMK9WzJ7Bhg2H/r7/K+oNhYTLlFQMqERUDwycRkb1zcbFcGFujAZYvt358eroMcFLl5gLjxwOXLsl2+bIsWJ6dLV0AmjY1ff6IEUBystxXa0vVGtPOnYEZMwzHnj0r/WJr1GAzPxEBYPgkIqp+1AWkVVotsHq16THZ2TLd1KVLEjCN9zdpIvv//Ve+PndONkCa+Y3DZ7t2QGqqnCMoyHQLDwcmTzYcGxcn86Yal42IqhyGTyIisuTsLFM91a1ruf+vv+R+drbMd3r5sqHGNDTUcGxmpgRb9di4ONlUgwebhs8WLWRuVU9PQ0ANDJTbdu2AiRMNx546JbWp/v6G1yAiu8DwSUREJaNOoJ/faH5XV6kJzcyUWtJr12RT7xvPpXr7tjT/AxJAz5yRTXXffabhs107GYylTnlVs6ZsAQHS9D9/vuHY338HvLwMx3DuVCKbYvgkIqLy5epqvRbVmJub9EVNSzMNqOrWuLHh2MxMOWdGhgysUleZOnFCHtfpTMPnwIGmfVyNg2iPHsDrrxse27ZNHq9VS2pd/fw4oIqojDF8EhFR5aDRSPDz8jINm+ZcXSVs5uTI7fXrpptx039WFtCwoexPTJTa1dRU2c6dk4CpUhRg1CgJtyqtVmpTAwNlHtU33jA89vnn0kUgMFDCqp9fmb0VRFUZwycREdknR0cJfsYB0pyLC3D0qNzX62WUvhpSExJMA2N2NtCpk2Ee1Rs3JKz++69sxt0LFAV47DGToOoEYLC7OxxDQ4F77gHefttw/LZt0kc1OBgICeGgKqrWGD6JiKh6cHCQZUh9fWXEvjkXF2DXLsPX2dlSW5qQIOHT29v0sd69DUH1338BnQ5OGRkyGKp5c8Ox6uIAWVmGfd7ehiDaowewZInhsb/+koFUwcGWK1wRVQEMn0RERNY4O0s4DAmxfMzFBYiMNHytKNAlJuL3L75AzyZN4Gg8if/t21KjGh8PXLkifVVTUmQ7edJ0wn9FkTCqBlUfH3n9mjUlkHbtCsydazj+p5+km0JAgDzu68vR/1TpMXwSERGVlkYD1KiBtNBQKL16AU5Ohsfc3YHffpP7iiL9TePjZZqqq1dNuw2kpclk/VevSkhNTpYtNtbwOipFAYYMMV3Z6k45EBAA9Otn2vS/apUM7FKDakCATGPl789BVVShGD6JiIgqikYjTe7e3sBdd1k+7uUFnD4twTIlxRBSr1+XwVV16hiOzcyUKafU0f7JyfK8mzdla9XKcKyiAPPmGaazMqbVylRW33xj2PfSS9IvVZ1nVb319WVQpVJj+CQiIqpsNBppcvfxsVzeVOXmZpjwH5Appm7cMIRRT0/DY9nZsoSq+lhSkvRVTUqSQGq8ipVeL31QrQVVR0dg6FBg61bDvpdfNoz6Dww0zLfq729aA0x0B8MnERFRVeDklP/ofxcXYN06y/06ndSq6vWGfdnZwMyZpvOt/vuvBNucHAmgKr0eeOEF60EVAO69F/j+e8PX06bJVFkBAabN/+p0VgEBJfveya5UmfCpKApycnKQm98HgCqETqeDo6MjMjMzK+W1cHJygpad8YmIhJOT5YAqV1fpH2ouO1tqSxXFdN/06aarVyUmSlBVFNPR+no98N57pkHXWJ8+wC+/GL7u31/KFxAgzf3e3tItwdtbVsfq399wbFyc1AR7e3MFKztQJcJndnY24uPjkZGRYeuiVHuKoiAoKAiXLl2CxrhjfCWh0WgQGhoKT+PmKCIiKpyzs+kE/oAEVeOJ91W5ucCtW6Y1ojk5wPLl0tSfmCib8f1atUyf//PPpkHX2D33mIbPNm3k9dRyqiHV21tmGnjvPcOxL78Mh+xsNLhyBRp1gFdQkExt5eVVnHeESsjuw6der8f58+eh1WoREhICZ2fnShl6qgu9Xo+0tDR4enrCoZJ1SlcUBdevX8fly5fRuHFj1oASEZUXrVaa1I05O5sue2rOOGgqCvD114YVrJKTZQBWaqrctmljeqx5bazarxWwbMpfsQLamzfRCrDsitCpE/DHH4avX35ZBlip4VS9DQjglFalYPfhMzs7G3q9HmFhYXDnZLw2p9frkZ2dDVdX10oXPgGgZs2auHDhAnQ6HcMnEVFlYlxx5OgIDBtW9OeptaxpaYY5VNWgat7SNWUKcpOScO3YMQRrtXC4dk1mFUhLM51zFQD+7/8k+JrTaqWbwI4dhn2rV0s3gRo15DXVzctLug0EBxft+6kG7D58qipj0KHKh7XiRERVlFZrmCGgIMuXQ6/T4WBkJAYPHgwHdUR+WposCKDS62WAlBpO4+PlfkKCBF1Hswi1ZIn1oAoAXboA+/YZvm7VSo41Dqnq1qSJ6YpXn38uNbvqwCx1c3Mr6jtT6VSZ8ElERERUYmr4Uzk4AK+8YnlcTo4EUJ3OsE+vB8aMkXCakiJB1ngz74Jw+bKhj6q5jh1Nw+f8+cClS5bHubsDHTqYLgn73//K6ljmQVWdWcA8MNtI5SgFERERkT1wdLScIcDBwXQ1qcL88Yd0CzAOqOrX5n1U+/SR0fxq/9fERAm+GRnSv9XYW2/JsdY0bgycOmX4evZsmZWgRg3ZfHwM92vVArp3Nxybm1umfVwZPm2kV69eaNu2LVavXm3rohAREVFFsra6VX4++sj0a3WJ1sRE06VVAWDCBODKFcMMAmpYvXHDMtR+9ZX1GlW1fCdOGL5u3x44e9Y0oKr369aVWQyKgeGTiIiIyF4YL9FqbulS68/JyZGaUmPLlkk4vXVLtuRkw33jZVwB2ZeeLtvVq6aPNWnC8ElERERERhwdLcPqhAlFf/7hw9ZDanKy6UICRVR1h4irCd3alplZ9GONR74VdGwp3Lx5E+PGjYOvry/c3d0xaNAgnD59Ou/xixcvYsiQIfD19YWHhwdatGiByMjIvOeOGTMGNWvWhJubGxo3bowNGzaUqjxEREREefz8gAYNpPm9d29g+HDgscek3+jkycU+XdWt+SxoBZvBg4EffjB8XauWZXW0qmdP05Fk9epJ/wlz+a3CUAQTJkzA6dOnsW3bNnh7e2PBggUYPHgwjh8/DicnJ0yfPh3Z2dn4/fff4eHhgePHj+et0PPcc8/h+PHj+PHHHxEQEIAzZ87gtnlgJiIiIqokqm74tBNq6Ny7dy+6du0KANi4cSPCwsLwzTff4MEHH0RcXBxGjhyJVq1aAQAaNGiQ9/y4uDi0a9cOERERAIB69epV+PdAREREVFRVN3ympeX/mPl0AQkJ+R9rPnn9hQslLpI1sbGxcHR0RKdOnfL2+fv746677kJsbCwAYNasWXjyySexY8cO3HPPPRg5ciRat24NAHjyyScxcuRIHDp0CP3798f999+fF2KJiIiIKpuq2+fTwyP/zdW16MearyCQ33ElpOTTXK8oSt5qPJMmTcK5c+cwduxYHD16FBEREXjrrbcAAIMGDcLFixcxe/ZsXL16FX379sXcuXNLXB4iIiKi8lSi8Ll27VrUr18frq6uCA8Px+7du/M99quvvkK/fv1Qs2ZNeHt7o0uXLti+fXuJC1zVNG/eHDk5Ofjzzz/z9iUlJeHUqVNo1qxZ3r6wsDBMnToVX331FZ555hl88MEHeY/VrFkTEyZMwKefforVq1fj/fffr9DvgYiIiKioih0+N2/ejNmzZ2Px4sU4fPgwunfvjkGDBiEunxn1f//9d/Tr1w+RkZGIjo5G7969MWTIEBw+fLjUha8KGjdujGHDhuGJJ57Anj17cOTIETz66KOoXbs2hg0bBgCYPXs2tm/fjvPnz+PQoUP49ddf84Lp888/j2+//RZnzpzBsWPH8P3335uEViIiIqLKpNjhc+XKlZg4cSImTZqEZs2aYfXq1QgLC8M777xj9fjVq1dj/vz56NChAxo3boxXXnkFjRs3xnfffVfqwlcVGzZsQHh4OO677z506dIFiqIgMjISTk5OAIDc3FxMnz4dzZo1w8CBA3HXXXdh7dq1AABnZ2csWrQIrVu3Ro8ePaDVavH555/b8tshIiIiylexBhxlZ2cjOjoaCxcuNNnfv39/7Nu3r0jn0Ov1SE1NhZ+fX77HZGVlISsrK+/rlJQUAIBOp4NOpzM5VqfTQVEU6PV66PX6on4rNvfrr78CkPfDx8cHH5kvn3XnMQB444038MYbb1h9/Nlnn8Wzzz6b73MrmtqHVb0mlY1er4eiKNDpdNCW4Tq19kj9LJl/pqjy4bWyD7xO9oPXqnwU9f0sVvhMTExEbm4uAgMDTfYHBgbi2rVrRTrHihUrkJ6ejoceeijfY5YvX46lVpaIioqKgrvZTPqOjo4ICgpCWloasrOzi1QGKn+pqam2LoJV2dnZuH37Nn7//XfkmK+JW03t3LnT1kWgIuK1sg+8TvaD16psZeQ3Z7qZEk21pI7CVhmPzC7Ipk2bsGTJEnz77beoVatWvsctWrQIc+bMyfs6JSUFYWFh6N27N/z9/U2OzczMxKVLl+Dp6QlX81HsVOEURUFqaiq8vLyK9DNR0TIzM+Hm5oYePXpU+58XnU6HnTt3ol+/fnldPKhy4rWyD7xO9oPXqnyoLdWFKVb4DAgIgFartajlTEhIsKgNNbd582ZMnDgRW7ZswT333FPgsS4uLnBxcbHY7+TkZPFDkpubC41GAwcHBziYz8lJFU5talevSWXj4OAAjUZj9WepuuJ7YT94rewDr5P94LUqW0V9L4uVDpydnREeHm5RTb1z584CJzbftGkTJkyYgM8++wz33ntvcV6SiIiIiKqQYje7z5kzB2PHjkVERAS6dOmC999/H3FxcZg6dSoAaTK/cuUKPv74YwASPMeNG4c33ngDnTt3zqs1dXNzg4+PTxl+K0RERERU2RU7fI4aNQpJSUlYtmwZ4uPj0bJlS0RGRqJu3boAgPj4eJM5P9977z3k5ORg+vTpmD59et7+8ePHWx3hTURERERVV4kGHE2bNg3Tpk2z+ph5oNy1a1dJXoKIiIiIqqDKNyKEiIiIiKoshk8iIiIiqjAMn0RERERUYRg+iYiIiKjCMHwau3wZiIqS22qIa9wSERFReau64TM9Pf8tM9Py2LVrgbp1gT595HbtWtl/+3bRzlsCP/30E+6++27UqFED/v7+uO+++3D27Nm8xy9fvozRo0fDz88PHh4eiIiIwJ9//pn3+LZt2xAREQFXV1cEBARgxIgReY9pNBp88803Jq9Xo0aNvNkILly4AI1Ggy+++AK9evWCq6srPv30UyQlJeHhhx9GaGgo3N3d0apVK2zatMnkPHq9Hq+99hoaNWoEFxcX1KlTBy+//DIA4J577sG8efNMjk9KSoKLiwt+/fXXEr1PREREVHVU3fDp6Zn/NnKk6bEBAcD06cCdpSGh18vXnp7AoEGmx9arZ/2cJZCeno45c+bgwIED+OWXX+Dg4IDhw4dDr9cjLS0NPXv2xNWrV7Ft2zYcOXIE8+fPz1u+8ocffsCIESNw77334vDhw/jll18QERFR7DIsWLAAs2bNQmxsLAYMGIDMzEyEh4fj+++/xz///IPJkydj7NixJqF30aJFeO211/Dcc8/h+PHj+Oyzz/KWV3388cfx5ZdfIisrK+/4jRs3IiQkBL179y7R+0RERERVR4nm+axyFMUmLzvSLASvW7cOtWrVwvHjx7Fv3z5cv34dBw4cgJ+fHwCgUaNGece+/PLLGD16NJYuXZq3r02bNsUuw+zZs01qTAFg7ty5efdnzpyJn376CVu2bEGnTp2QmpqKN954A2vWrMH48eMBAA0bNsTdd9+d9z3NmjUL3377LUaPHg0A2LBhAyZMmACNRlPs8hEREVHVUnXDZ1pa/o9ptaZf//030KyZoeZTPeb4cSAszPTYCxfKrIhnz57Fc889hz/++AOJiYl5tZpxcXGIiYlBu3bt8oKnuZiYGDzxxBOlLoN5bWlubi5effVVbN68GVeuXEFWVhaysrLg4eEBAIiNjUVWVhb69u1r9XwuLi546KGHsGHDBowePRoxMTE4cuSIRRcAIiIiqp6qbvi8E5aKpEkT4P33gSlTgNxcCZ7vvSf7S3PeQgwZMgRhYWH44IMPEBISAr1ej5YtWyI7Oxtubm4FPrewxzUaDRSzGl1rA4o8zL6fFStWYNWqVVi9ejVatWoFDw8PzJ49G9nZ2UV6XQAYO3YsevTogcuXL2P9+vXo27dv3vKrREREVL1V3T6fxTVxotRqRkXJ7cSJ5fpySUlJiI2NxX/+8x/07dsXzZo1w82bN/Meb926NWJiYnDjxg2rz2/dujV++eWXfM9fs2ZNxMfH5319+vRpZGRkFFqu3bt3Y9iwYXj00UfRpk0bNGjQAKdPn857vHHjxnBzcyvwtVu0aIGIiAh88MEH+Oyzz/D4448X+rpERERUPVTdms+SCA2VrQL4+vrC398f77//PoKDgxEXF4eFCxfmPf7www/jlVdewf3334/ly5cjODgYhw8fRkhICLp06YIXXngBffv2RcOGDTF69Gjk5OTgxx9/xPz58wEAffr0wZo1a9C5c2fo9XosWLAATk5OhZarUaNG2Lp1K/bt2wdfX1+sXLkS165dQ7NmzQAArq6uWLBgAebPnw9nZ2d069YN169fx7FjxzDRKLA//vjjmDVrFtzd3TF8+PAyfveIiIjIXrHm00YcHBzw+eefIzo6Gi1btsTTTz+N//u//8t73NnZGTt27ECtWrUwePBgtGrVCq+++iq0d/qr9urVC1u2bMG2bdvQtm1b9OnTx2RE+ooVKxAWFoYePXrgkUcewdy5c+Hu7l5ouZ577jm0b98eAwYMQK9evRAUFIT777/f4phnnnkGzz//PJo1a4ZRo0YhISHB5JiHH34Yjo6OeOSRR+Dq6lqKd4qIiIiqEtZ82tA999yD48ePm+wz7qdZt25dfPnll/k+f8SIERYj1VUhISHYvn27yb5bt27l3a9Xr55Fn1AA8PPzK3RwkIODAxYvXozFixfne8zNmzeRmZlpUhtKRERExJpPKlM6nQ6XLl3CwoUL0blzZ7Rv397WRSIiIqJKhDWfVKb27t2Lvn37okmTJgXW2hIREVH1xPBJZapXr164efMmvL294eDAinUiIiIyxXRARERERBWG4ZOIiIiIKgzDJxERERFVGIZPIiIiIqowDJ9EREREVGEYPomIiIiowjB82rF69eph9erVRTpWo9EUunIRERERUXlj+CQiIiKiCsPwSUREREQVpsqFT0UB0tNtsylK0cv53nvvoXbt2tDr9Sb7hw4divHjx+Ps2bMYNmwYAgMD4enpiQ4dOuDnn38us/fp6NGj6NOnD9zc3ODv74/JkycjLS0t7/Fdu3ahY8eO8PDwQI0aNdCtWzdcvHgRAHDkyBH07t0bXl5e8Pb2Rnh4OA4ePFhmZSMiIqKqq8qFz4wMwNPTNltGRtHL+eCDDyIxMRFRUVF5+27evInt27djzJgxSEtLw+DBg/Hzzz/j8OHDGDBgAIYMGYK4uLgyeI8yMHDgQPj6+uLAgQPYsmULfv75Z8yYMQMAkJOTg/vvvx89e/bE33//jf3792Py5MnQaDQAgDFjxiA0NBQHDhxAdHQ0Fi5cCCcnp1KXi4iIiKo+ru1uI35+fhg4cCA+++wz9O3bFwCwZcsW+Pn5oW/fvtBqtWjTpk3e8S+99BK+/vprbNu2LS8kltTGjRtx+/ZtfPzxx/Dw8AAArFmzBkOGDMFrr70GJycnJCcn47777kPDhg0BAM2aNct7flxcHObNm4emTZsCABo3blyq8hAREVH1UeVqPt3dgbQ022zu7sUr65gxY7B161ZkZWUBkFA4evRoaLVapKenY/78+WjevDlq1KgBT09PnDhxokxqPmNjY9GmTZu84AkA3bp1g16vx8mTJ+Hn54cJEybk1ba+8cYbiI+Pzzt2zpw5mDRpEu655x68+uqrOHv2bKnLRERERNVDlQufGg3g4WGb7U6rdJENGTIEer0eP/zwAy5duoTdu3fj0UcfBQDMmzcPW7duxcsvv4zdu3cjJiYGrVq1QnZ2dqnfI0VR8prQzan7N2zYgP3796Nr167YvHkzmjRpgj/++AMAsGTJEhw7dgz33nsvfv31VzRv3hxff/11qctFREREVV+VC5/2xM3NDSNGjMDGjRuxadMmNGnSBOHh4QCA3bt3Y8KECRg+fDhatWqFoKAgXLhwoUxet3nz5oiJiUF6enrevr1798LBwQFNmjTJ29euXTssWrQI+/btQ8uWLfHZZ5/lPdakSRM8/fTT2LFjB0aMGIENGzaUSdmIiIioamP4tLExY8bghx9+wPr16/NqPQGgUaNG+OqrrxATE4MjR47gkUcesRgZX5rXdHV1xfjx4/HPP/8gKioKM2fOxNixYxEYGIjz589j0aJF2L9/Py5evIgdO3bg1KlTaNasGW7fvo0ZM2Zg165duHjxIvbu3YsDBw6Y9AklIiIiyg8HHNlYnz594Ofnh5MnT+KRRx7J279q1So8/vjj6Nq1KwICArBgwQKkpKSUyWu6u7tj+/bteOqpp9ChQwe4u7tj5MiRWLlyZd7jJ06cwP/+9z8kJSUhODgYM2bMwJQpU5CTk4OkpCSMGzcO//77LwICAjBixAgsXbq0TMpGREREVRvDp41ptVpcvXrVYn+9evXw66+/muybPn26ydfFaYZXzCYhbdWqlcX5VYGBgfn24XR2dsamTZuK/LpERERExtjsTkREREQVhuGzCti4cSM8PT2tbi1atLB18YiIiIjysNm9Chg6dCg6depk9TGuPERERESVCcNnFeDl5QUvLy9bF4OIiIioUGx2JyIiIqIKw/BJRERERBWG4ZOIiIiIKgzDJxERERFVGIZPIiIiIqowDJ92rF69eli9erWti0FERERUZAyfRg4eBPr0kVsiIiIiKnsMn0Y+/hiIigI++cTWJan6cnNzodfrbV0MIiIiqmBVLnwqCpCeXvQtNhbYswfYuxf4/HM5x6ZN8vWePfJ4Uc+lKEUv53vvvYfatWtbBLChQ4di/PjxOHv2LIYNG4bAwEB4enqiQ4cO+Pnnn0v8vqxcuRKtWrWCh4cHwsLCMG3aNKSlpZkcs3fvXvTs2RPu7u7w9fXFgAEDcPPmTQCAXq/Ha6+9hkaNGsHFxQV16tTByy+/DADYtWsXNBoNbt26lXeumJgYaDQaXLhwAQDw0UcfoUaNGvj+++/RvHlzuLi44OLFizhw4AD69euHgIAA+Pj4oGfPnjh06JBJuW7duoXJkycjMDAQrq6uaNmyJb7//nukp6fD29sbX375pcnx3333HTw8PJCamlri94uIiIjKR5Vb4SgjA/D0LN05rl8H7r67+M9LSwM8PIp27IMPPohZs2YhKioKffv2BQDcvHkT27dvx3fffYe0tDQMHjwYL730ElxdXfG///0PQ4YMwcmTJ1GnTp1il83BwQFvvvkm6tWrh/Pnz2PatGmYP38+1q5dC0DCYt++ffH444/jzTffhKOjI6KiopCbmwsAWLRoET744AOsWrUKd999N+Lj43HixIlilSEjIwPLly/Hhx9+CH9/f9SqVQvnz5/H+PHj8eabbwIAVqxYgcGDB+P06dPw8vKCXq/HoEGDkJqaik8//RQNGzbE8ePHodVq4eHhgdGjR2PDhg144IEH8l5H/ZqrPhEREVVCih1ITk5WACiJiYkWj92+fVs5fvy4cvv2bUVRFCUtTVGkDrLit7S04n1fQ4cOVR5//PG8r9977z0lKChIycnJsXp88+bNlbfeeivv67p16yqrVq0q3ove8cUXXyj+/v55Xz/88MNKt27drB6bkpKiuLi4KB988IHVx6OiohQAys2bN5Xc3Fzl5s2bSnR0tAJAOX/+vKIoirJhwwYFgBITE1NguXJychQvLy/lu+++UxRFUbZv3644ODgoJ0+etHr8n3/+qWi1WuXKlSuKoijK9evXFScnJ2XXrl1Wjzf/eanOsrOzlW+++UbJzs62dVGoELxW9oHXyX7wWpUPNa8lJycXeFyVa3Z3d5cayOJse/ZYP9eePcU7j7t78co6ZswYbN26FVlZWQCAjRs3YvTo0dBqtUhPT8f8+fPRvHlz1KhRA56enjhx4gTi4uJK9L5ERUWhX79+qF27Nry8vDBu3DgkJSUhPT0dgKHm05rY2FhkZWXl+3hROTs7o3Xr1ib7EhISMHXqVDRp0gQ+Pj7w8fFBWlpa3vcZExOD0NBQNGnSxOo5O3bsiBYtWuDjjz8GAHzyySeoU6cOevToUaqyEhERUfmocuFTo5Gm7+Jsbm7yXAcH01s3t+KdR6MpXlmHDBkCvV6PH374AZcuXcLu3bvx6KOPAgDmzZuHrVu34uWXX8bu3bsRExODVq1aITs7u9jvycWLFzF48GC0bNkSW7duRXR0NN5++20AgE6nu/O9uuX7/IIeA6RJHwAUo06v6nnNz6Mxe5MmTJiA6OhorF69Gvv27UNMTAz8/f3zvs/CXhsAJk2ahA0bNgCQJvfHHnvM4nWIiIiocqhy4bMkatUCgoKA8HDg3XflNihI9pcnNzc3jBgxAhs3bsSmTZvQpEkThIeHAwB2796NCRMmYPjw4WjVqhWCgoLyBu8U18GDB5GTk4MVK1agc+fOaNKkCa5evWpyTOvWrfHLL79YfX7jxo3h5uaW7+M1a9YEAMTHx+fti4mJKVLZdu/ejVmzZmHw4MFo0aIFXFxckJiYaFKuy5cv49SpU/me49FHH0VcXBzefPNNHDt2DOPHjy/SaxMREVHFq3IDjkoiNBS4cAFwdpbay8mTgexswMWl/F97zJgxGDJkCI4dO5ZX6wkAjRo1wldffYUhQ4ZAo9HgueeeK/HURA0bNkROTg7eeustDBkyBHv37sW7775rcsyiRYvQqlUrTJs2DVOnToWzszOioqLw4IMPIiAgAAsWLMD8+fPh7OyMbt264fr16zh27BgmTpyIRo0aISwsDEuWLMGyZctw5MgRrFq1qkhla9SoET755BNEREQgJSUF8+bNM6nt7NmzJ3r06IGRI0di5cqVaNSoEU6cOAGNRoOBAwcCAHx9fTFixAjMmzcP/fv3R2hoaIneJyIiIip/rPm8w8XF0Gyu0VRM8ASAPn36wM/PDydPnsQjjzySt3/VqlXw9fVF165dMWTIEAwYMADt27cv0Wu0bdsWK1euxGuvvYaWLVti48aNWL58uckxTZo0wY4dO3DkyBF07NgRXbp0wbfffgtHR/n75LnnnsMzzzyD559/Hs2aNcOoUaOQkJAAAHBycsKmTZtw4sQJtGvXDm+88QaWLVtWpLKtX78eN2/eRLt27TB27FjMmjULtcyqnLdu3YoOHTrg4YcfRvPmzTF//vy8UfiqiRMnIjs7G48//niJ3iMiIiKqGBrFuKNeJZWSkgIfHx8kJibC39/f5LHMzEycP38e9evXh6urq41KSCq9Xo+UlBR4e3vn9QWtCBs3bsRTTz2Fq1evwtnZOd/j+PNioNPpEBkZicGDB8PJycnWxaEC8FrZB14n+8FrVT7UvJacnAxvb+98j2OzO9m1jIwMnD9/HsuXL8eUKVMKDJ5ERERke2x2rwI2btwIT09Pq1uLFi1sXbxy9frrr6Nt27YIDAzEokWLbF0cIiIiKgRrPquAoUOHolOnTlYfq+rNCUuWLMGSJUtsXQwiIiIqIobPKsDLy4tLSRIREZFdqDLN7nYwbooqAf6cEBER2Zbdh0+1WTkjI8PGJSF7oK6cpNVqbVwSIiKi6snum921Wi1q1KiRN+eku7s7l1a0Ib1ej+zsbGRmZlboVEtFodfrcf36dbi7u+fNX0pEREQVq0r8Bg4KCgKAvABKtqMoCm7fvm11HffKwMHBAXXq1KmUZSMiIqoOqkT41Gg0CA4ORq1ataDT6WxdnGpNp9Ph999/R48ePSrlSHtnZ+dKVyNLRERUnVSJ8KnSarXsy2djWq0WOTk5cHV1rZThk4iIiGyrRFVAa9euzVueMDw8HLt37y7w+N9++w3h4eFwdXVFgwYN8O6775aosERERERk34odPjdv3ozZs2dj8eLFOHz4MLp3745BgwYhLi7O6vHnz5/H4MGD0b17dxw+fBjPPvssZs2aha1bt5a68ERERERkX4odPleuXImJEydi0qRJaNasGVavXo2wsDC88847Vo9/9913UadOHaxevRrNmjXDpEmT8Pjjj+O///1vqQtPRERERPalWH0+s7OzER0djYULF5rs79+/P/bt22f1Ofv370f//v1N9g0YMADr1q2DTqez2i8wKysLWVlZeV8nJycDAG7cuFGc4pIN6HQ6ZGRkICkpiX0+KzleK/vBa2UfeJ3sB69V+UhNTQVQ+IIuxQqfiYmJyM3NRWBgoMn+wMBAXLt2zepzrl27ZvX4nJwcJCYmIjg42OI5y5cvx9KlSy32N2nSpDjFJSIiIqIKlpqaCh8fn3wfL9Fod/M5EhVFKXDeRGvHW9uvWrRoEebMmZP39a1bt1C3bl3ExcUV+M2Q7aWkpCAsLAyXLl2Ct7e3rYtDBeC1sh+8VvaB18l+8FqVD0VRkJqaipCQkAKPK1b4DAgIgFartajlTEhIsKjdVAUFBVk93tHREf7+/laf4+LiAhcXF4v9Pj4+/CGxE97e3rxWdoLXyn7wWtkHXif7wWtV9opSSVisAUfOzs4IDw/Hzp07Tfbv3LkTXbt2tfqcLl26WBy/Y8cOREREsJ8FERERUTVT7NHuc+bMwYcffoj169cjNjYWTz/9NOLi4jB16lQA0mQ+bty4vOOnTp2KixcvYs6cOYiNjcX69euxbt06zJ07t+y+CyIiIiKyC8Xu8zlq1CgkJSVh2bJliI+PR8uWLREZGYm6desCAOLj403m/Kxfvz4iIyPx9NNP4+2330ZISAjefPNNjBw5ssiv6eLighdeeMFqUzxVLrxW9oPXyn7wWtkHXif7wWtlWxqlsPHwRERERERlpETLaxIRERERlQTDJxERERFVGIZPIiIiIqowDJ9EREREVGEqffhcu3Yt6tevD1dXV4SHh2P37t22LhKZWbJkCTQajckWFBRk62IRgN9//x1DhgxBSEgINBoNvvnmG5PHFUXBkiVLEBISAjc3N/Tq1QvHjh2zTWGrucKu1YQJEyw+Z507d7ZNYau55cuXo0OHDvDy8kKtWrVw//334+TJkybH8LNle0W5Tvxc2UalDp+bN2/G7NmzsXjxYhw+fBjdu3fHoEGDTKZyosqhRYsWiI+Pz9uOHj1q6yIRgPT0dLRp0wZr1qyx+vjrr7+OlStXYs2aNThw4ACCgoLQr18/pKamVnBJqbBrBQADBw40+ZxFRkZWYAlJ9dtvv2H69On4448/sHPnTuTk5KB///5IT0/PO4afLdsrynUC+LmyCaUS69ixozJ16lSTfU2bNlUWLlxooxKRNS+88ILSpk0bWxeDCgFA+frrr/O+1uv1SlBQkPLqq6/m7cvMzFR8fHyUd9991wYlJJX5tVIURRk/frwybNgwm5SHCpaQkKAAUH777TdFUfjZqqzMr5Oi8HNlK5W25jM7OxvR0dHo37+/yf7+/ftj3759NioV5ef06dMICQlB/fr1MXr0aJw7d87WRaJCnD9/HteuXTP5jLm4uKBnz578jFVSu3btQq1atdCkSRM88cQTSEhIsHWRCEBycjIAwM/PDwA/W5WV+XVS8XNV8Spt+ExMTERubi4CAwNN9gcGBuLatWs2KhVZ06lTJ3z88cfYvn07PvjgA1y7dg1du3ZFUlKSrYtGBVA/R/yM2YdBgwZh48aN+PXXX7FixQocOHAAffr0QVZWlq2LVq0pioI5c+bg7rvvRsuWLQHws1UZWbtOAD9XtlLs5TUrmkajMflaURSLfWRbgwYNyrvfqlUrdOnSBQ0bNsT//vc/zJkzx4Ylo6LgZ8w+jBo1Ku9+y5YtERERgbp16+KHH37AiBEjbFiy6m3GjBn4+++/sWfPHovH+NmqPPK7Tvxc2UalrfkMCAiAVqu1+CsxISHB4q9Jqlw8PDzQqlUrnD592tZFoQKoMxLwM2afgoODUbduXX7ObGjmzJnYtm0boqKiEBoamrefn63KJb/rZA0/VxWj0oZPZ2dnhIeHY+fOnSb7d+7cia5du9qoVFQUWVlZiI2NRXBwsK2LQgWoX78+goKCTD5j2dnZ+O233/gZswNJSUm4dOkSP2c2oCgKZsyYga+++gq//vor6tevb/I4P1uVQ2HXyRp+ripGpW52nzNnDsaOHYuIiAh06dIF77//PuLi4jB16lRbF42MzJ07F0OGDEGdOnWQkJCAl156CSkpKRg/fryti1btpaWl4cyZM3lfnz9/HjExMfDz80OdOnUwe/ZsvPLKK2jcuDEaN26MV155Be7u7njkkUdsWOrqqaBr5efnhyVLlmDkyJEIDg7GhQsX8OyzzyIgIADDhw+3Yamrp+nTp+Ozzz7Dt99+Cy8vr7waTh8fH7i5uUGj0fCzVQkUdp3S0tL4ubIVG460L5K3335bqVu3ruLs7Ky0b9/eZIoEqhxGjRqlBAcHK05OTkpISIgyYsQI5dixY7YuFimKEhUVpQCw2MaPH68oikwJ88ILLyhBQUGKi4uL0qNHD+Xo0aO2LXQ1VdC1ysjIUPr376/UrFlTcXJyUurUqaOMHz9eiYuLs3WxqyVr1wmAsmHDhrxj+NmyvcKuEz9XtqNRFEWpyLBLRERERNVXpe3zSURERERVD8MnEREREVUYhk8iIiIiqjAMn0RERERUYRg+iYiIiKjCMHwSERERUYVh+CQiIiKiCsPwSUREREQVhuGTiMiOaDQafPPNN7YuBhFRiTF8EhEV0YQJE6DRaCy2gQMH2rpoRER2w9HWBSAisicDBw7Ehg0bTPa5uLjYqDRERPaHNZ9ERMXg4uKCoKAgk83X1xeANIm/8847GDRoENzc3FC/fn1s2bLF5PlHjx5Fnz594ObmBn9/f0yePBlpaWkmx6xfvx4tWrSAi4sLgoODMWPGDJPHExMTMXz4cLi7u6Nx48bYtm1b+X7TRERliOGTiKgMPffccxg5ciSOHDmCRx99FA8//DBiY2MBABkZGRg4cCB8fX1x4MABbNmyBT///LNJuHznnXcwffp0TJ48GUePHsW2bdvQqFEjk9dYunQpHnroIfz9998YPHgwxowZgxs3blTo90lEVFIaRVEUWxeCiMgeTJgwAZ9++ilcXV1N9i9YsADPPfccNBoNpk6dinfeeSfvsc6dO6N9+/ZYu3YtPvjgAyxYsACXLl2Ch4cHACAyMhJDhgzB1atXERgYiNq1a+Oxxx7DSy+9ZLUMGo0G//nPf/Diiy8CANLT0+Hl5YXIyEj2PSUiu8A+n0RExdC7d2+TcAkAfn5+efe7dOli8liXLl0QExMDAIiNjUWbNm3ygicAdOvWDXq9HidPnoRGo8HVq1fRt2/fAsvQunXrvPseHh7w8vJCQkJCSb8lIqIKxfBJRFQMHh4eFs3ghdFoNAAARVHy7ls7xs3NrUjnc3JysniuXq8vVpmIiGyFfT6JiMrQH3/8YfF106ZNAQDNmzdHTEwM0tPT8x7fu3cvHBwc0KRJE3h5eaFevXr45ZdfKrTMREQViTWfRETFkJWVhWvXrpnsc3R0REBAAABgy5YtiIiIwN13342NGzfir7/+wrp16wAAY8aMwQsvvIDx48djyZIluH79OmbOnImxY8ciMDAQALBkyRJMnToVtWrVwqBBg5Camoq9e/di5syZFfuNEhGVE4ZPIqJi+OmnnxAcHGyy76677sKJEycAyEj0zz//HNOmTUNQUBA2btyI5s2bAwDc3d2xfft2PPXUU+jQoQPc3d0xcuRIrFy5Mu9c48ePR2ZmJlatWoW5c+ciICAADzzwQMV9g0RE5Yyj3YmIyohGo8HXX3+N+++/39ZFISKqtNjnk4iIiIgqDMMnEREREVUY9vkkIioj7MVERFQ41nwSERERUYVh+CQiIiKiCsPwSUREREQVhuGTiIiIiCoMwycRERERVRiGTyIiIiKqMAyfRERERFRhGD6JiIiIqML8P/a/ZmH4oQAfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(history.history).plot(\n",
    "    figsize=(8,5), xlim=[0, 29], ylim=[0,1], grid=True, xlabel=\"Epoch\",\n",
    "    style=[\"r--\", \"r--.\", \"b-\", \"b-*\"]\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "638f243d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 4ms/step - loss: 0.3245 - accuracy: 0.8841\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.32452696561813354, 0.8841000199317932]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#You can see that both the training accuracy and the validation \n",
    "#accuracy steadily increase during training, while the training\n",
    "#loss and the validation loss decrease. This is good. The validation \n",
    "#curves are relatively close to each other at first, but they get\n",
    "#further apart over time, which shows that there’s a little bit of \n",
    "#overfitting. In this particular case, the model looks like it\n",
    "#performed better on the validation set than on the training set\n",
    "#at the beginning of training, but that’s not actually the case.\n",
    "#The validation error is computed at the end of each epoch, while\n",
    "#the training error is computed using a running mean during each \n",
    "#epoch, so the training curve should be shifted by half an epoch \n",
    "#to the left. If you do that, you will see that the training and \n",
    "#validation curves overlap almost perfectly at the beginning of training.\n",
    "\n",
    "\n",
    "#The training set performance ends up beating the validation performance,\n",
    "#as is generally the case when you train for long enough. You can tell\n",
    "#that the model has not quite converged yet, as the validation loss is\n",
    "#still going down, so you should probably continue training. This is \n",
    "#as simple as calling the fit() method again, since Keras just continues \n",
    "#training where it left off: you should be able to reach about 89.8% \n",
    "#validation accuracy, while the training accuracy will continue to rise\n",
    "#up to 100% (this is not always the case).\n",
    "\n",
    "\n",
    "#If you are not satisfied with the performance of your model, you \n",
    "#should go back and tune the hyperparameters. The first one to check \n",
    "#is the learning rate. If that doesn’t help, try another optimizer \n",
    "#(and always retune the learning rate after changing any hyperparameter).\n",
    "#If the performance is still not great, then try tuning model \n",
    "#hyperparameters such as the number of layers, the number of neurons per \n",
    "#layer, and the types of activation functions to use for each hidden \n",
    "#layer. You can also try tuning other hyperparameters, such as the \n",
    "#batch size (it can be set in the fit() method using the batch_size\n",
    "#argument, which defaults to 32). We will get back to hyperparameter\n",
    "#tuning at the end of this chapter. Once you are satisfied with your \n",
    "#model’s validation accuracy, you should evaluate it on the test set\n",
    "#to estimate the generalization error before you deploy the model to\n",
    "#production. You can easily do this using the evaluate() method (it\n",
    "#also supports several other arguments, such as batch_size and \n",
    "#sample_weight; please check the documentation for more details):\n",
    "\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "929052ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "#As you saw in Chapter 2, it is common to get slightly lower performance\n",
    "#on the test set than on the validation set, because the hyperparameters\n",
    "#are tuned on the validation set, not the test set (however, in this \n",
    "#example, we did not do any hyperparameter tuning, so the lower \n",
    "#accuracy is just bad luck). Remember to resist the temptation to \n",
    "#tweak the hyperparameters on the test set, or else your estimate \n",
    "#of the generalization error will be too optimistic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d769012f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 58ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:29:22.203535: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.98],\n",
       "       [0.  , 0.  , 1.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  ],\n",
       "       [0.  , 1.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Using the model to make predictions\n",
    "#Now let’s use the model’s predict() method to make predictions on new \n",
    "#instances. Since we don’t have actual new instances, we’ll just use \n",
    "#the first three instances of the test set:\n",
    "\n",
    "X_new = X_test[:3]\n",
    "y_proba = model.predict(X_new)\n",
    "y_proba.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0265aef5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9, 2, 1])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For each instance the model estimates one probability per class, \n",
    "#from class 0 to class 9. This is similar to the output of the \n",
    "#predict_proba() method in Scikit-Learn classifiers. For example, for the first \n",
    "#image it estimates that the probability of class 9 (ankle boot) is 96%, \n",
    "#the probability of class 7 (sneaker) is 2%, the probability of class 5 (sandal)\n",
    "#is 1%, and the probabilities of the other classes are negligible. In other words,\n",
    "#it is highly confident that the first image is footwear, most likely ankle boots\n",
    "#but possibly sneakers or sandals. If you only care about the class with the\n",
    "#highest estimated probability (even if that probability is quite low), then\n",
    "#you can use the argmax() method to get the highest probability class index \n",
    "#for each instance:\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "y_pred = y_proba.argmax(axis=-1) #-1 returns same as axis=1 because the last axis *IS* axis=1\n",
    "y_pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "504101ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Ankle boot', 'Pullover', 'Trouser'], dtype='<U11')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(class_names)[y_pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "197700f5",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "only integer scalar arrays can be converted to a scalar index",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [32], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m np\u001b[38;5;241m.\u001b[39marray(\u001b[43mclass_names\u001b[49m\u001b[43m[\u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m]\u001b[49m)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#explanation of error:\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m#Perhaps the error message is somewhat misleading, but the gist \u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m#is that class_names is a list, not a numpy array. You cannot use array \u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m#indexing on it. Make it an array first with np.array()\u001b[39;00m\n",
      "\u001b[0;31mTypeError\u001b[0m: only integer scalar arrays can be converted to a scalar index"
     ]
    }
   ],
   "source": [
    "np.array(class_names[y_pred])\n",
    "\n",
    "#explanation of error:\n",
    "#Perhaps the error message is somewhat misleading, but the gist \n",
    "#is that class_names is a list, not a numpy array. You cannot use array \n",
    "#indexing on it. Make it an array first with np.array()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a651d04b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "69952703",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9, 2, 1], dtype=uint8)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Here, the classifier actually classified all three images \n",
    "#correctly (these images are shown in Figure 10-12):\n",
    "\n",
    "y_new = y_test[:3]\n",
    "y_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4d2854e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:29:29.407188: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      " 11/363 [..............................] - ETA: 1s - loss: 4.8937 - root_mean_squared_error: 2.2122  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:29:30.176358: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - ETA: 0s - loss: 0.9042 - root_mean_squared_error: 0.9509"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:29:32.208758: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 3s 6ms/step - loss: 0.9042 - root_mean_squared_error: 0.9509 - val_loss: 0.3798 - val_root_mean_squared_error: 0.6163\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3769 - root_mean_squared_error: 0.6139 - val_loss: 1.6157 - val_root_mean_squared_error: 1.2711\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3605 - root_mean_squared_error: 0.6004 - val_loss: 0.4478 - val_root_mean_squared_error: 0.6692\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3415 - root_mean_squared_error: 0.5844 - val_loss: 1.3377 - val_root_mean_squared_error: 1.1566\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3340 - root_mean_squared_error: 0.5780 - val_loss: 1.2113 - val_root_mean_squared_error: 1.1006\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3270 - root_mean_squared_error: 0.5718 - val_loss: 0.3213 - val_root_mean_squared_error: 0.5668\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3116 - root_mean_squared_error: 0.5582 - val_loss: 1.0822 - val_root_mean_squared_error: 1.0403\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3057 - root_mean_squared_error: 0.5529 - val_loss: 1.1842 - val_root_mean_squared_error: 1.0882\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3026 - root_mean_squared_error: 0.5501 - val_loss: 3.0613 - val_root_mean_squared_error: 1.7497\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3226 - root_mean_squared_error: 0.5680 - val_loss: 0.3328 - val_root_mean_squared_error: 0.5769\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2928 - root_mean_squared_error: 0.5411 - val_loss: 0.4476 - val_root_mean_squared_error: 0.6690\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3012 - root_mean_squared_error: 0.5488 - val_loss: 0.3894 - val_root_mean_squared_error: 0.6240\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2847 - root_mean_squared_error: 0.5336 - val_loss: 0.2791 - val_root_mean_squared_error: 0.5283\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2835 - root_mean_squared_error: 0.5325 - val_loss: 0.3460 - val_root_mean_squared_error: 0.5882\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2820 - root_mean_squared_error: 0.5310 - val_loss: 0.2923 - val_root_mean_squared_error: 0.5406\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2798 - root_mean_squared_error: 0.5289 - val_loss: 0.3072 - val_root_mean_squared_error: 0.5543\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2771 - root_mean_squared_error: 0.5264 - val_loss: 0.2986 - val_root_mean_squared_error: 0.5464\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2715 - root_mean_squared_error: 0.5210 - val_loss: 0.2959 - val_root_mean_squared_error: 0.5440\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2707 - root_mean_squared_error: 0.5202 - val_loss: 0.2751 - val_root_mean_squared_error: 0.5245\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.2726 - root_mean_squared_error: 0.5221 - val_loss: 0.4272 - val_root_mean_squared_error: 0.6536\n"
     ]
    }
   ],
   "source": [
    "#Now you know how to use the sequential API to build, train, and \n",
    "#evaluate a classification MLP. But what about regression?\n",
    "\n",
    "#Building a Regression MLP Using the Sequential API\n",
    "\n",
    "#Let’s switch back to the California housing problem and tackle it \n",
    "#using the same MLP as earlier, with 3 hidden layers composed of 50 \n",
    "#neurons each, but this time building it with Keras.\n",
    "\n",
    "#Using the sequential API to build, train, evaluate, and use a \n",
    "#regression MLP is quite similar to what we did for classification. \n",
    "#The main differences in the following code example are the fact that\n",
    "#the output layer has a single neuron (since we only want to predict a\n",
    "#single value) and it uses no activation function, the loss function\n",
    "#is the mean squared error, the metric is the RMSE, and we’re using an\n",
    "#Adam optimizer like Scikit-Learn’s MLPRegressor did. Moreover, in this\n",
    "#example we don’t need a Flatten layer, and instead we’re using a \n",
    "#Normalization layer as the first layer: it does the same thing as\n",
    "#Scikit-Learn’s StandardScaler, but it must be fitted to the training\n",
    "#data using its adapt() method before you call the model’s fit() method.\n",
    "#(Keras has other preprocessing layers, which will be covered in \n",
    "#Chapter 13). Let’s take a look:\n",
    "\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state=42)\n",
    "\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "norm_layer = tf.keras.layers.Normalization(input_shape=X_train.shape[1:])\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Dense(50, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(50, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(50, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(1)\n",
    "])\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
    "model.compile(loss=\"mse\", optimizer=optimizer, metrics=\n",
    "             [\"RootMeanSquaredError\"])\n",
    "norm_layer.adapt(X_train)\n",
    "history = model.fit(X_train, y_train, epochs=20,\n",
    "                   validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "da6aeabd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8,)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "090fa829",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11610, 8)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d3687aa8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "362.8125"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "11610 / 32 #there are ~363 batches per epoch, each having 32 instances - this is default batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "24e8046c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 1s 4ms/step - loss: 0.2841 - root_mean_squared_error: 0.5330\n"
     ]
    }
   ],
   "source": [
    "mse_test, rmse_test = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f3d267cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 39ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:30:48.730994: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.48932123],\n",
       "       [1.1363262 ],\n",
       "       [5.086301  ]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new = X_test[:3]\n",
    "y_pred = model.predict(X_new)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "20431662",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.477  , 0.458  , 5.00001])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "dcc82409",
   "metadata": {},
   "outputs": [],
   "source": [
    "#As you can see, the sequential API is quite clean and straightforward.\n",
    "#However, although Sequential models are extremely common, it is \n",
    "#sometimes useful to build neural networks with more complex topologies,\n",
    "#or with multiple inputs or outputs. For this purpose, Keras offers the \n",
    "#functional API.\n",
    "\n",
    "\n",
    "#Building Complex Models Using the Functional API\n",
    "\n",
    "#One example of a nonsequential neural network is a Wide & Deep neural\n",
    "#network. This neural network architecture was introduced in a 2016\n",
    "#paper by Heng-Tze Cheng et al.15 It connects all or part of the inputs\n",
    "#directly to the output layer, as shown in Figure 10-13. This \n",
    "#architecture makes it possible for the neural network to learn both\n",
    "#deep patterns (using the deep path) and simple rules (through the \n",
    "#short path).16 \n",
    "\n",
    "#In contrast, a regular MLP forces all the data to \n",
    "#flow through the full stack of layers; thus, simple patterns in the \n",
    "#data may end up being distorted by this sequence of transformations.\n",
    "\n",
    "\n",
    "#Let’s build such a neural network to tackle the California housing problem:\n",
    "\n",
    "\n",
    "normalization_layer = tf.keras.layers.Normalization()\n",
    "hidden_layer1 = tf.keras.layers.Dense(30, activation=\"relu\")\n",
    "hidden_layer2 = tf.keras.layers.Dense(30, activation=\"relu\")\n",
    "concat_layer = tf.keras.layers.Concatenate()\n",
    "output_layer = tf.keras.layers.Dense(1)\n",
    "\n",
    "input_ = tf.keras.layers.Input(shape=X_train.shape[1:])\n",
    "normalized = normalization_layer(input_)\n",
    "hidden1 = hidden_layer1(normalized)\n",
    "hidden2 = hidden_layer2(hidden1)\n",
    "concat = concat_layer([normalized, hidden2])\n",
    "output = output_layer(concat)\n",
    "\n",
    "model = tf.keras.Model(inputs=[input_], outputs=[output])\n",
    "\n",
    "#At a high level, the first five lines create all the layers we need to build the model, the \n",
    "#next six lines use these layers just like functions to go from the input to the output, and \n",
    "#the last line creates a Keras Model object by pointing to the input and the output. Let’s go\n",
    "#through this code in more detail:\n",
    "\n",
    "\n",
    "#First, we create five layers: a Normalization layer to standardize the inputs, two Dense layers with \n",
    "#30 neurons each, using the ReLU activation function, a Concatenate layer, and one more Dense layer \n",
    "#with a single neuron for the output layer, without any activation function.\n",
    "\n",
    "\n",
    "#Next, we create an Input object (the variable name input_ is used to avoid overshadowing Python’s \n",
    "#built-in input() function). This is a specification of the kind of input the model will get, including\n",
    "#its shape and optionally its dtype, which defaults to 32-bit floats. A model may actually have multiple\n",
    "#inputs, as you will see shortly.\n",
    "\n",
    "\n",
    "\n",
    "#Then we use the Normalization layer just like a function, passing it the Input object. This is why \n",
    "#this is called the functional API. Note that we are just telling Keras how it should connect the \n",
    "#layers together; no actual data is being processed yet, as the Input object is just a data specification.\n",
    "#In other words, it’s a symbolic input. The output of this call is also symbolic: normalized doesn’t store\n",
    "#any actual data, it’s just used to construct the model.\n",
    "\n",
    "\n",
    "#In the same way, we then pass normalized to hidden_layer1, which outputs hidden1, and we pass hidden1 to\n",
    "#hidden_layer2, which outputs hidden2.\n",
    "\n",
    "\n",
    "#So far we’ve connected the layers sequentially, but then we use the concat_layer to concatenate the input \n",
    "#and the second hidden layer’s output. Again, no actual data is concatenated yet: it’s all symbolic, to \n",
    "#build the model.\n",
    "\n",
    "\n",
    "#Then we pass concat to the output_layer, which gives us the final output.\n",
    "\n",
    "#Lastly, we create a Keras Model, specifying which inputs and outputs to use.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0846de52",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Once you have built this Keras model, everything is exactly like earlier, so there’s no need to\n",
    "#repeat it here: you compile the model, adapt the Normalization layer, fit the model, evaluate it, and\n",
    "#use it to make predictions.\n",
    "\n",
    "#But what if you want to send a subset of the features through the wide path and a different subset \n",
    "#(possibly overlapping) through the deep path, as illustrated in Figure 10-14? In this case, one \n",
    "#solution is to use multiple inputs. For example, suppose we want to send five features through \n",
    "#the wide path (features 0 to 4), and six features through the deep path (features 2 to 7). We \n",
    "#can do this as follows:\n",
    "\n",
    "\n",
    "input_wide = tf.keras.layers.Input(shape=[5])\n",
    "input_deep = tf.keras.layers.Input(shape=[6])\n",
    "norm_layer_wide = tf.keras.layers.Normalization()\n",
    "norm_layer_deep = tf.keras.layers.Normalization()\n",
    "norm_wide = norm_layer_wide(input_wide)\n",
    "norm_deep = norm_layer_deep(input_deep)\n",
    "hidden1 = tf.keras.layers.Dense(30, activation=\"relu\")(norm_deep)\n",
    "hidden2 = tf.keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
    "\n",
    "concat = tf.keras.layers.concatenate([norm_wide, hidden2])\n",
    "output = tf.keras.layers.Dense(1)(concat)\n",
    "model = tf.keras.Model(inputs=[input_wide, input_deep], outputs=[output])\n",
    "\n",
    "\n",
    "#There are a few things to note in this example, compared to the previous one:\n",
    "\n",
    "#Each Dense layer is created and called on the same line. This is a common practice, as it\n",
    "#makes the code more concise without losing clarity. However, we can’t do this with the \n",
    "#Normalization layer since we need a reference to the layer to be able to call its adapt()\n",
    "#method before fitting the model.\n",
    "\n",
    "#We used tf.keras.layers.concatenate(), which creates a Concatenate layer and calls it with the given inputs.\n",
    "\n",
    "#We specified inputs=[input_wide, input_deep] when creating the model, since there are two inputs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ea5dc752",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:31:00.005803: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2023-03-31 10:31:00.675136: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      " 21/363 [>.............................] - ETA: 1s - loss: 4.9541 - root_mean_squared_error: 2.2258 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:31:01.439300: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361/363 [============================>.] - ETA: 0s - loss: 1.3297 - root_mean_squared_error: 1.1531"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:31:03.407442: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 2s 6ms/step - loss: 1.3260 - root_mean_squared_error: 1.1515 - val_loss: 0.8780 - val_root_mean_squared_error: 0.9370\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.4709 - root_mean_squared_error: 0.6862 - val_loss: 0.5435 - val_root_mean_squared_error: 0.7373\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.4168 - root_mean_squared_error: 0.6456 - val_loss: 0.4754 - val_root_mean_squared_error: 0.6895\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3985 - root_mean_squared_error: 0.6312 - val_loss: 0.3651 - val_root_mean_squared_error: 0.6042\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3835 - root_mean_squared_error: 0.6193 - val_loss: 0.3442 - val_root_mean_squared_error: 0.5867\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3679 - root_mean_squared_error: 0.6065 - val_loss: 0.3664 - val_root_mean_squared_error: 0.6053\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3604 - root_mean_squared_error: 0.6003 - val_loss: 0.3364 - val_root_mean_squared_error: 0.5800\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3520 - root_mean_squared_error: 0.5933 - val_loss: 0.3642 - val_root_mean_squared_error: 0.6035\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3460 - root_mean_squared_error: 0.5882 - val_loss: 0.6417 - val_root_mean_squared_error: 0.8011\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3476 - root_mean_squared_error: 0.5895 - val_loss: 0.4149 - val_root_mean_squared_error: 0.6441\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3369 - root_mean_squared_error: 0.5805 - val_loss: 0.3382 - val_root_mean_squared_error: 0.5815\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3377 - root_mean_squared_error: 0.5811 - val_loss: 0.6372 - val_root_mean_squared_error: 0.7983\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3359 - root_mean_squared_error: 0.5796 - val_loss: 0.4373 - val_root_mean_squared_error: 0.6613\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3308 - root_mean_squared_error: 0.5751 - val_loss: 0.5665 - val_root_mean_squared_error: 0.7527\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3302 - root_mean_squared_error: 0.5746 - val_loss: 1.0557 - val_root_mean_squared_error: 1.0275\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3322 - root_mean_squared_error: 0.5764 - val_loss: 2.0239 - val_root_mean_squared_error: 1.4226\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3357 - root_mean_squared_error: 0.5794 - val_loss: 1.0361 - val_root_mean_squared_error: 1.0179\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3273 - root_mean_squared_error: 0.5721 - val_loss: 0.3699 - val_root_mean_squared_error: 0.6082\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3188 - root_mean_squared_error: 0.5646 - val_loss: 0.3267 - val_root_mean_squared_error: 0.5716\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3212 - root_mean_squared_error: 0.5667 - val_loss: 0.3092 - val_root_mean_squared_error: 0.5560\n"
     ]
    }
   ],
   "source": [
    "#Now we can compile the model as usual, but when we call the fit() method, instead of passing a single\n",
    "#input matrix X_train, we must pass a pair of matrices (X_train_wide, X_train_deep), one per input. The\n",
    "#same is true for X_valid, and also for X_test and X_new when you call evaluate() or predict():\n",
    "\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
    "model.compile(loss=\"mse\", optimizer=optimizer, metrics=[\"RootMeanSquaredError\"])\n",
    "\n",
    "X_train_wide, X_train_deep = X_train[:, :5], X_train[:, 2:]\n",
    "X_valid_wide, X_valid_deep = X_valid[:, :5], X_valid[:, 2:]\n",
    "X_test_wide, X_test_deep = X_test[:, :5], X_test[:, 2:]\n",
    "X_new_wide, X_new_deep = X_test_wide[:3], X_test_deep[:3]\n",
    "\n",
    "norm_layer_wide.adapt(X_train_wide)\n",
    "norm_layer_deep.adapt(X_train_deep)\n",
    "\n",
    "history = model.fit((X_train_wide, X_train_deep), y_train, epochs=20,\n",
    "                   validation_data=((X_valid_wide, X_valid_deep), y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "53d8d347",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TIP\n",
    "#Instead of passing a tuple (X_train_wide, X_train_deep), you can pass a dictionary \n",
    "#{\"input_wide\": X_train_wide, \"input_deep\": X_train_deep}, if you set name=\"input_wide\" and name=\"input_deep\" when\n",
    "#creating the inputs. This is highly recommended when there are many inputs, to clarify the code and avoid \n",
    "#getting the order wrong.\n",
    "\n",
    "#There are also many use cases in which you may want to have multiple outputs:\n",
    "\n",
    "\n",
    "#The task may demand it. For instance, you may want to locate and classify the main object in a picture. \n",
    "#This is both a regression tasks and a classification task.\n",
    "\n",
    "#Similarly, you may have multiple independent tasks based on the same data. Sure, you could train one neural \n",
    "#network per task, but in many cases you will get better results on all tasks by training a single neural \n",
    "#network with one output per task. This is because the neural network can learn features in the data that \n",
    "#are useful across tasks. For example, you could perform multitask classification on pictures of faces, \n",
    "#using one output to classify the person’s facial expression (smiling, surprised, etc.) and another output\n",
    "#to identify whether they are wearing glasses or not.\n",
    "\n",
    "#Another use case is as a regularization technique (i.e., a training constraint whose objective is to \n",
    "#reduce overfitting and thus improve the model’s ability to generalize). For example, you may want to add\n",
    "#an auxiliary output in a neural network architecture (see Figure 10-15) to ensure that the underlying part\n",
    "#of the network learns something useful on its own, without relying on the rest of the network.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "02e840fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "  7/363 [..............................] - ETA: 2s - loss: 1.0591 - dense_15_loss: 0.3485 - dense_16_loss: 7.4541 - dense_15_root_mean_squared_error: 0.5904 - dense_16_root_mean_squared_error: 2.7302  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:32:36.226693: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - ETA: 0s - loss: 0.4878 - dense_15_loss: 0.3339 - dense_16_loss: 1.8732 - dense_15_root_mean_squared_error: 0.5779 - dense_16_root_mean_squared_error: 1.3687"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:32:39.154398: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 4s 9ms/step - loss: 0.4878 - dense_15_loss: 0.3339 - dense_16_loss: 1.8732 - dense_15_root_mean_squared_error: 0.5779 - dense_16_root_mean_squared_error: 1.3687 - val_loss: 0.7339 - val_dense_15_loss: 0.3721 - val_dense_16_loss: 3.9907 - val_dense_15_root_mean_squared_error: 0.6100 - val_dense_16_root_mean_squared_error: 1.9977\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3793 - dense_15_loss: 0.3308 - dense_16_loss: 0.8162 - dense_15_root_mean_squared_error: 0.5752 - dense_16_root_mean_squared_error: 0.9034 - val_loss: 0.5624 - val_dense_15_loss: 0.3518 - val_dense_16_loss: 2.4572 - val_dense_15_root_mean_squared_error: 0.5932 - val_dense_16_root_mean_squared_error: 1.5675\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3568 - dense_15_loss: 0.3249 - dense_16_loss: 0.6442 - dense_15_root_mean_squared_error: 0.5700 - dense_16_root_mean_squared_error: 0.8026 - val_loss: 0.4363 - val_dense_15_loss: 0.3088 - val_dense_16_loss: 1.5835 - val_dense_15_root_mean_squared_error: 0.5557 - val_dense_16_root_mean_squared_error: 1.2584\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3465 - dense_15_loss: 0.3224 - dense_16_loss: 0.5638 - dense_15_root_mean_squared_error: 0.5678 - dense_16_root_mean_squared_error: 0.7508 - val_loss: 0.3840 - val_dense_15_loss: 0.3091 - val_dense_16_loss: 1.0583 - val_dense_15_root_mean_squared_error: 0.5560 - val_dense_16_root_mean_squared_error: 1.0287\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3393 - dense_15_loss: 0.3204 - dense_16_loss: 0.5098 - dense_15_root_mean_squared_error: 0.5660 - dense_16_root_mean_squared_error: 0.7140 - val_loss: 0.3520 - val_dense_15_loss: 0.3092 - val_dense_16_loss: 0.7379 - val_dense_15_root_mean_squared_error: 0.5560 - val_dense_16_root_mean_squared_error: 0.8590\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3308 - dense_15_loss: 0.3154 - dense_16_loss: 0.4696 - dense_15_root_mean_squared_error: 0.5616 - dense_16_root_mean_squared_error: 0.6853 - val_loss: 0.3319 - val_dense_15_loss: 0.2992 - val_dense_16_loss: 0.6258 - val_dense_15_root_mean_squared_error: 0.5470 - val_dense_16_root_mean_squared_error: 0.7911\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3290 - dense_15_loss: 0.3161 - dense_16_loss: 0.4448 - dense_15_root_mean_squared_error: 0.5623 - dense_16_root_mean_squared_error: 0.6669 - val_loss: 0.5525 - val_dense_15_loss: 0.5486 - val_dense_16_loss: 0.5880 - val_dense_15_root_mean_squared_error: 0.7407 - val_dense_16_root_mean_squared_error: 0.7668\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3245 - dense_15_loss: 0.3132 - dense_16_loss: 0.4268 - dense_15_root_mean_squared_error: 0.5596 - dense_16_root_mean_squared_error: 0.6533 - val_loss: 0.7252 - val_dense_15_loss: 0.7316 - val_dense_16_loss: 0.6675 - val_dense_15_root_mean_squared_error: 0.8553 - val_dense_16_root_mean_squared_error: 0.8170\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3230 - dense_15_loss: 0.3129 - dense_16_loss: 0.4138 - dense_15_root_mean_squared_error: 0.5594 - dense_16_root_mean_squared_error: 0.6433 - val_loss: 0.5947 - val_dense_15_loss: 0.6024 - val_dense_16_loss: 0.5251 - val_dense_15_root_mean_squared_error: 0.7761 - val_dense_16_root_mean_squared_error: 0.7246\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3222 - dense_15_loss: 0.3131 - dense_16_loss: 0.4039 - dense_15_root_mean_squared_error: 0.5596 - dense_16_root_mean_squared_error: 0.6355 - val_loss: 0.4281 - val_dense_15_loss: 0.4261 - val_dense_16_loss: 0.4453 - val_dense_15_root_mean_squared_error: 0.6528 - val_dense_16_root_mean_squared_error: 0.6673\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3163 - dense_15_loss: 0.3077 - dense_16_loss: 0.3938 - dense_15_root_mean_squared_error: 0.5547 - dense_16_root_mean_squared_error: 0.6276 - val_loss: 0.3043 - val_dense_15_loss: 0.2954 - val_dense_16_loss: 0.3841 - val_dense_15_root_mean_squared_error: 0.5435 - val_dense_16_root_mean_squared_error: 0.6198\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3183 - dense_15_loss: 0.3102 - dense_16_loss: 0.3910 - dense_15_root_mean_squared_error: 0.5570 - dense_16_root_mean_squared_error: 0.6253 - val_loss: 0.4336 - val_dense_15_loss: 0.4326 - val_dense_16_loss: 0.4425 - val_dense_15_root_mean_squared_error: 0.6577 - val_dense_16_root_mean_squared_error: 0.6652\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3147 - dense_15_loss: 0.3069 - dense_16_loss: 0.3849 - dense_15_root_mean_squared_error: 0.5540 - dense_16_root_mean_squared_error: 0.6204 - val_loss: 0.3700 - val_dense_15_loss: 0.3650 - val_dense_16_loss: 0.4146 - val_dense_15_root_mean_squared_error: 0.6042 - val_dense_16_root_mean_squared_error: 0.6439\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3128 - dense_15_loss: 0.3054 - dense_16_loss: 0.3791 - dense_15_root_mean_squared_error: 0.5527 - dense_16_root_mean_squared_error: 0.6157 - val_loss: 0.3672 - val_dense_15_loss: 0.3452 - val_dense_16_loss: 0.5651 - val_dense_15_root_mean_squared_error: 0.5875 - val_dense_16_root_mean_squared_error: 0.7517\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3128 - dense_15_loss: 0.3058 - dense_16_loss: 0.3757 - dense_15_root_mean_squared_error: 0.5530 - dense_16_root_mean_squared_error: 0.6129 - val_loss: 0.3593 - val_dense_15_loss: 0.3580 - val_dense_16_loss: 0.3713 - val_dense_15_root_mean_squared_error: 0.5983 - val_dense_16_root_mean_squared_error: 0.6094\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3091 - dense_15_loss: 0.3023 - dense_16_loss: 0.3706 - dense_15_root_mean_squared_error: 0.5498 - dense_16_root_mean_squared_error: 0.6087 - val_loss: 0.3713 - val_dense_15_loss: 0.3627 - val_dense_16_loss: 0.4490 - val_dense_15_root_mean_squared_error: 0.6022 - val_dense_16_root_mean_squared_error: 0.6701\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3107 - dense_15_loss: 0.3041 - dense_16_loss: 0.3693 - dense_15_root_mean_squared_error: 0.5515 - dense_16_root_mean_squared_error: 0.6077 - val_loss: 0.3208 - val_dense_15_loss: 0.3164 - val_dense_16_loss: 0.3603 - val_dense_15_root_mean_squared_error: 0.5625 - val_dense_16_root_mean_squared_error: 0.6003\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3069 - dense_15_loss: 0.3006 - dense_16_loss: 0.3638 - dense_15_root_mean_squared_error: 0.5483 - dense_16_root_mean_squared_error: 0.6032 - val_loss: 0.5547 - val_dense_15_loss: 0.5547 - val_dense_16_loss: 0.5547 - val_dense_15_root_mean_squared_error: 0.7448 - val_dense_16_root_mean_squared_error: 0.7448\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3061 - dense_15_loss: 0.2998 - dense_16_loss: 0.3627 - dense_15_root_mean_squared_error: 0.5475 - dense_16_root_mean_squared_error: 0.6022 - val_loss: 0.4903 - val_dense_15_loss: 0.4927 - val_dense_16_loss: 0.4688 - val_dense_15_root_mean_squared_error: 0.7019 - val_dense_16_root_mean_squared_error: 0.6847\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3081 - dense_15_loss: 0.3022 - dense_16_loss: 0.3612 - dense_15_root_mean_squared_error: 0.5497 - dense_16_root_mean_squared_error: 0.6010 - val_loss: 0.6393 - val_dense_15_loss: 0.6648 - val_dense_16_loss: 0.4094 - val_dense_15_root_mean_squared_error: 0.8153 - val_dense_16_root_mean_squared_error: 0.6398\n"
     ]
    }
   ],
   "source": [
    "[...] # Same as above, up to the main output layer output = tf.keras.layers.Dense(1)(concat)\n",
    "aux_output = tf.keras.layers.Dense(1)(hidden2)\n",
    "model = tf.keras.Model(inputs=[input_wide, input_deep],\n",
    "                          outputs=[output, aux_output])\n",
    "\n",
    "\n",
    "#Each output will need its own loss function. Therefore, when we compile the model, we should pass a list \n",
    "#of losses. If we pass a single loss, Keras will assume that the same loss must be used for all outputs. \n",
    "#By default, Keras will compute all the losses and simply add them up to get the final loss used for \n",
    "#training. Since we care much more about the main output than about the auxiliary output (as it is just \n",
    "#used for regularization), we want to give the main output’s loss a much greater weight. Luckily, it is \n",
    "#possible to set all the loss weights when compiling the model:\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
    "model.compile(loss=(\"mse\", \"mse\"), loss_weights=(0.9, 0.1),\n",
    "             optimizer=optimizer, metrics=[\"RootMeanSquaredError\"])\n",
    "\n",
    "#TIP:\n",
    "#Instead of passing a tuple loss=(\"mse\", \"mse\"), you can pass a dictionary loss= {\"output\": \"mse\",\n",
    "#\"aux_output\": \"mse\"}, assuming you created the output layers with name=\"output\" and name=\"aux_output\".\n",
    "#Just like for the inputs, this clarifies the code and avoids errors when there are several outputs. You\n",
    "#can also pass a dictionary for loss_weights.\n",
    "\n",
    "\n",
    "#Now when we train the model, we need to provide labels for each output. In this example, \n",
    "#the main output and the auxiliary output should try to predict the same thing, so they should\n",
    "#use the same labels. So instead of passing y_train, we need to pass (y_train, y_train), or a \n",
    "#dictionary {\"output\": y_train, \"aux_output\": y_train} if the outputs were named \"output\" and \"aux_output\".\n",
    "#The same goes for y_valid and y_test:\n",
    "\n",
    "\n",
    "norm_layer_wide.adapt(X_train_wide)\n",
    "norm_layer_deep.adapt(X_train_deep)\n",
    "\n",
    "history = model.fit(\n",
    "        (X_train_wide, X_train_deep), (y_train, y_train), epochs=20,\n",
    "        validation_data=((X_valid_wide, X_valid_deep), (y_valid, y_valid))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "dd220c75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 1s 6ms/step - loss: 0.3120 - dense_15_loss: 0.3064 - dense_16_loss: 0.3620 - dense_15_root_mean_squared_error: 0.5535 - dense_16_root_mean_squared_error: 0.6017\n",
      "0.3119716942310333 0.30641406774520874 0.36198946833610535 0.5535468459129333 0.6016556024551392\n"
     ]
    }
   ],
   "source": [
    "#When we evaluate the model, Keras returns the weighted sum of the losses, as well as all the \n",
    "#individual losses and metrics:\n",
    "\n",
    "eval_results = model.evaluate((X_test_wide, X_test_deep), (y_test, y_test))\n",
    "weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse = eval_results\n",
    "\n",
    "\n",
    "print(weighted_sum_of_losses, main_loss, aux_loss, main_rmse, aux_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "5854ef79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 48ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:33:54.125010: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    }
   ],
   "source": [
    "#Similarly, the predict() method will return predictions for each output:\n",
    "\n",
    "y_pred_main, y_pred_aux = model.predict((X_new_wide, X_new_deep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ceed3202",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 15ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'dense_15': array([[0.3386169],\n",
       "        [1.1068047],\n",
       "        [3.5085928]], dtype=float32),\n",
       " 'dense_16': array([[0.58676076],\n",
       "        [0.9867044 ],\n",
       "        [3.2857347 ]], dtype=float32)}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The predict() method returns a tuple, and it does not have a return_dict argument to get a dictionary instead.\n",
    "#However, you can create one using model.output_names:\n",
    "\n",
    "y_pred_tuple = model.predict((X_new_wide, X_new_deep))\n",
    "y_pred = dict(zip(model.output_names, y_pred_tuple))\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7dbc5575",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_keras_model/assets\n"
     ]
    }
   ],
   "source": [
    "#Saving and Restoring a Model\n",
    "#Saving a trained Keras model is as simple as it gets:\n",
    "\n",
    "model.save(\"my_keras_model\", save_format=\"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "52a095d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#When you set save_format=\"tf\",18 Keras saves the model using \n",
    "#TensorFlow’s SavedModel format: this is a directory (with the \n",
    "#given name) containing several files and subdirectories. In \n",
    "#particular, the saved_model.pb file contains the model’s \n",
    "#architecture and logic in the form of a serialized computation \n",
    "#graph, so you don’t need to deploy the model’s source code in \n",
    "#order to use it in production; the SavedModel is sufficient \n",
    "#(you will see how this works in Chapter 12). The keras_metadata.pb \n",
    "#file contains extra information needed by Keras. The variables \n",
    "#subdirectory contains all the parameter values (including the \n",
    "#connection weights, the biases, the normalization statistics, \n",
    "#and the optimizer’s parameters), possibly split across multiple\n",
    "#files if the model is very large. Lastly, the assets directory\n",
    "#may contain extra files, such as data samples, feature names,\n",
    "#class names, and so on. By default, the assets directory is empty.\n",
    "#Since the optimizer is also saved, including its hyperparameters\n",
    "#and any state it may have, after loading the model you can continue\n",
    "#training if you want."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "4af361b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 41ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:33:58.749083: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.3386169],\n",
       "        [1.1068047],\n",
       "        [3.5085928]], dtype=float32),\n",
       " array([[0.58676076],\n",
       "        [0.9867044 ],\n",
       "        [3.2857347 ]], dtype=float32))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#You will typically have a script that trains a model and saves it,\n",
    "#and one or more scripts (or web services) that load the model and use\n",
    "#it to evaluate it or to make predictions. Loading the model is just as\n",
    "#easy as saving it:\n",
    "\n",
    "model = tf.keras.models.load_model(\"my_keras_model/\")\n",
    "y_pred_main, y_pred_aux = model.predict((X_new_wide, X_new_deep))\n",
    "y_pred_main, y_pred_aux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ed50dbce",
   "metadata": {},
   "outputs": [],
   "source": [
    "#You can also use save_weights() and load_weights() to save and load\n",
    "#only the parameter values. This includes the connection weights, biases,\n",
    "#preprocessing stats, optimizer state, etc. The parameter values are\n",
    "#saved in one or more files such as my_weights.data-00004-of-00052, \n",
    "#plus an index file like my_weights.index.\n",
    "#Saving just the weights is faster and uses less disk space than \n",
    "#saving the whole model, so it’s perfect to save quick checkpoints\n",
    "#during training. If you’re training a big model, and it takes hours \n",
    "#or days, then you must save checkpoints regularly in case the computer\n",
    "#crashes. But how can you tell the fit() method to save checkpoints?\n",
    "#Use callbacks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "a689873c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Failed to find data adapter that can handle input: (<class 'list'> containing values of types {\"<class 'ellipsis'>\"}), <class 'NoneType'>",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [54], line 11\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#Using Callbacks\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m#The fit() method accepts a callbacks argument that lets you specify \u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#a list of objects that Keras will call before and after training, \u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m#checkpoints of your model at regular intervals during training, \u001b[39;00m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m#by default at the end of each epoch:\u001b[39;00m\n\u001b[1;32m      9\u001b[0m checkpoint_cb \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mModelCheckpoint(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmy_checkpoints\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     10\u001b[0m                                                   save_weights_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m---> 11\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mcheckpoint_cb\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/homl3/lib/python3.10/site-packages/keras/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m~/miniforge3/envs/homl3/lib/python3.10/site-packages/keras/engine/data_adapter.py:1083\u001b[0m, in \u001b[0;36mselect_data_adapter\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m   1080\u001b[0m adapter_cls \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mcls\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01min\u001b[39;00m ALL_ADAPTER_CLS \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mcan_handle(x, y)]\n\u001b[1;32m   1081\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m adapter_cls:\n\u001b[1;32m   1082\u001b[0m     \u001b[38;5;66;03m# TODO(scottzhu): This should be a less implementation-specific error.\u001b[39;00m\n\u001b[0;32m-> 1083\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1084\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to find data adapter that can handle \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1085\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(_type_name(x), _type_name(y))\n\u001b[1;32m   1086\u001b[0m     )\n\u001b[1;32m   1087\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(adapter_cls) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   1088\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m   1089\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mData adapters should be mutually exclusive for \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1090\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhandling inputs. Found multiple adapters \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m to handle \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1091\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(adapter_cls, _type_name(x), _type_name(y))\n\u001b[1;32m   1092\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Failed to find data adapter that can handle input: (<class 'list'> containing values of types {\"<class 'ellipsis'>\"}), <class 'NoneType'>"
     ]
    }
   ],
   "source": [
    "#Using Callbacks\n",
    "#The fit() method accepts a callbacks argument that lets you specify \n",
    "#a list of objects that Keras will call before and after training, \n",
    "#before and after each epoch, and even before and after processing \n",
    "#each batch. For example, the ModelCheckpoint callback saves \n",
    "#checkpoints of your model at regular intervals during training, \n",
    "#by default at the end of each epoch:\n",
    "\n",
    "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"my_checkpoints\",\n",
    "                                                  save_weights_only=True)\n",
    "history = model.fit([...], callbacks=[checkpoint_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5a7b384b",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Failed to find data adapter that can handle input: (<class 'list'> containing values of types {\"<class 'ellipsis'>\"}), <class 'NoneType'>",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [55], line 25\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#Moreover, if you use a validation set during training, you can set \u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m#save_ best_only=True when creating the ModelCheckpoint. In this case, \u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#it will only save your model when its performance on the validation set\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;66;03m#early when there is no more progress, to avoid wasting time and \u001b[39;00m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;66;03m#resources and to reduce overfitting:\u001b[39;00m\n\u001b[1;32m     22\u001b[0m early_stopping_cb \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mEarlyStopping(patience\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m,\n\u001b[1;32m     23\u001b[0m                                                     restore_best_weights\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m---> 25\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mcheckpoint_cb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mearly_stopping_cb\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;66;03m#The number of epochs can be set to a large value since training will\u001b[39;00m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;66;03m#stop automatically when there is no more progress (just make sure the\u001b[39;00m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;66;03m#learning rate is not too small, or else it might keep making slow\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;66;03m#Now let’s take a look at one more tool you should definitely \u001b[39;00m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;66;03m#have in your toolbox when using Keras: TensorBoard.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniforge3/envs/homl3/lib/python3.10/site-packages/keras/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m~/miniforge3/envs/homl3/lib/python3.10/site-packages/keras/engine/data_adapter.py:1083\u001b[0m, in \u001b[0;36mselect_data_adapter\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m   1080\u001b[0m adapter_cls \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mcls\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01min\u001b[39;00m ALL_ADAPTER_CLS \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mcan_handle(x, y)]\n\u001b[1;32m   1081\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m adapter_cls:\n\u001b[1;32m   1082\u001b[0m     \u001b[38;5;66;03m# TODO(scottzhu): This should be a less implementation-specific error.\u001b[39;00m\n\u001b[0;32m-> 1083\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1084\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to find data adapter that can handle \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1085\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(_type_name(x), _type_name(y))\n\u001b[1;32m   1086\u001b[0m     )\n\u001b[1;32m   1087\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(adapter_cls) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   1088\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m   1089\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mData adapters should be mutually exclusive for \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1090\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhandling inputs. Found multiple adapters \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m to handle \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1091\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(adapter_cls, _type_name(x), _type_name(y))\n\u001b[1;32m   1092\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Failed to find data adapter that can handle input: (<class 'list'> containing values of types {\"<class 'ellipsis'>\"}), <class 'NoneType'>"
     ]
    }
   ],
   "source": [
    "#Moreover, if you use a validation set during training, you can set \n",
    "#save_ best_only=True when creating the ModelCheckpoint. In this case, \n",
    "#it will only save your model when its performance on the validation set\n",
    "#is the best so far. This way, you do not need to worry about training\n",
    "#for too long and overfitting the training set: simply restore the last\n",
    "#saved model after training, and this will be the best model on the\n",
    "#validation set. This is one way to implement early stopping (introduced\n",
    "#in Chapter 4), but it won’t actually stop training.\n",
    "\n",
    "\n",
    "#Another way is to use the EarlyStopping callback. It will interrupt \n",
    "#training when it measures no progress on the validation set for a \n",
    "#number of epochs (defined by the patience argument), and if you set\n",
    "#restore_best_weights=True it will roll back to the best model at the\n",
    "#end of training. You can combine both callbacks to save checkpoints \n",
    "#of your model in case your computer crashes, and interrupt training \n",
    "#early when there is no more progress, to avoid wasting time and \n",
    "#resources and to reduce overfitting:\n",
    "\n",
    "\n",
    "\n",
    "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=10,\n",
    "                                                    restore_best_weights=True)\n",
    "\n",
    "history = model.fit([...], callbacks=[checkpoint_cb, early_stopping_cb])\n",
    "\n",
    "\n",
    "#The number of epochs can be set to a large value since training will\n",
    "#stop automatically when there is no more progress (just make sure the\n",
    "#learning rate is not too small, or else it might keep making slow\n",
    "#progress until the end). The EarlyStopping callback will store the\n",
    "#weights of the best model in RAM, and it will restore them for you\n",
    "#at the end of training.\n",
    "\n",
    "#Now let’s take a look at one more tool you should definitely \n",
    "#have in your toolbox when using Keras: TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "0f1f5040",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using TensorBoard for Visualization\n",
    "#TensorBoard is a great interactive visualization tool that you can use\n",
    "#to view the learning curves during training, compare curves and metrics\n",
    "#between multiple runs, visualize the computation graph, analyze training\n",
    "#statistics, view images generated by your model, visualize complex \n",
    "#multidimensional data projected down to 3D and automatically clustered\n",
    "#for you, profile your network (i.e., measure its speed to identify \n",
    "#bottlenecks), and more!\n",
    "\n",
    "#To use TensorBoard, you must modify your program so that it outputs\n",
    "#the data you want to visualize to special binary logfiles called event \n",
    "#files. Each binary data record is called a summary. The TensorBoard \n",
    "#server will monitor the log directory, and it will automatically pick\n",
    "#up the changes and update the visualizations: this allows you to \n",
    "#visualize live data (with a short delay), such as the learning \n",
    "#curves during training. In general, you want to point the TensorBoard\n",
    "#server to a root log directory and configure your program so that\n",
    "#it writes to a different subdirectory every time it runs. This way, \n",
    "#the same TensorBoard server instance will allow you to visualize and\n",
    "#compare data from multiple runs of your program, without getting \n",
    "#everything mixed up.\n",
    "\n",
    "#Let’s name the root log directory my_logs, and let’s define a little\n",
    "#function that generates the path of the log subdirectory based on the\n",
    "#current date and time, so that it’s different at every run:\n",
    "\n",
    "from pathlib import Path\n",
    "from time import strftime\n",
    "\n",
    "def get_run_logdir(root_logdir = \"my_logs\"):\n",
    "   return Path(root_logdir) / strftime(\"run_%Y_%m_%d_%H_%M_%S\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "eeccb13b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'run_2023_03_31_14_17_11'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strftime(\"run_%Y_%m_%d_%H_%M_%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "d4416aa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('my_logs/run_2023_03_31_14_17_11')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_logdir = get_run_logdir()\n",
    "run_logdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "45274f43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:17:12.590616: I tensorflow/core/profiler/lib/profiler_session.cc:101] Profiler session initializing.\n",
      "2023-03-31 14:17:12.590666: I tensorflow/core/profiler/lib/profiler_session.cc:116] Profiler session started.\n",
      "2023-03-31 14:17:12.591750: I tensorflow/core/profiler/lib/profiler_session.cc:128] Profiler session tear down.\n",
      "2023-03-31 14:17:12.696908: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124/363 [=========>....................] - ETA: 1s - loss: 0.6141 - dense_15_loss: 0.2883 - dense_16_loss: 0.3259 - dense_15_root_mean_squared_error: 0.5369 - dense_16_root_mean_squared_error: 0.5708"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:17:13.836309: I tensorflow/core/profiler/lib/profiler_session.cc:101] Profiler session initializing.\n",
      "2023-03-31 14:17:13.836325: I tensorflow/core/profiler/lib/profiler_session.cc:116] Profiler session started.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "213/363 [================>.............] - ETA: 2s - loss: 0.6099 - dense_15_loss: 0.2863 - dense_16_loss: 0.3237 - dense_15_root_mean_squared_error: 0.5350 - dense_16_root_mean_squared_error: 0.5689"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:17:16.707912: I tensorflow/core/profiler/lib/profiler_session.cc:67] Profiler session collecting data.\n",
      "2023-03-31 14:17:16.726048: I tensorflow/core/profiler/lib/profiler_session.cc:128] Profiler session tear down.\n",
      "2023-03-31 14:17:16.737340: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16\n",
      "\n",
      "2023-03-31 14:17:16.749543: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.trace.json.gz\n",
      "2023-03-31 14:17:16.771260: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16\n",
      "\n",
      "2023-03-31 14:17:16.773693: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.memory_profile.json.gz\n",
      "2023-03-31 14:17:16.774902: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16\n",
      "Dumped tool data for xplane.pb to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.xplane.pb\n",
      "Dumped tool data for overview_page.pb to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.overview_page.pb\n",
      "Dumped tool data for input_pipeline.pb to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.input_pipeline.pb\n",
      "Dumped tool data for tensorflow_stats.pb to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.tensorflow_stats.pb\n",
      "Dumped tool data for kernel_stats.pb to my_logs/run_2023_03_31_14_17_11/plugins/profile/2023_03_31_14_17_16/Rokos-MacBook-Pro-2.local.kernel_stats.pb\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - ETA: 0s - loss: 0.6245 - dense_15_loss: 0.2890 - dense_16_loss: 0.3355 - dense_15_root_mean_squared_error: 0.5376 - dense_16_root_mean_squared_error: 0.5792"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:17:18.140419: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 6s 16ms/step - loss: 0.6245 - dense_15_loss: 0.2890 - dense_16_loss: 0.3355 - dense_15_root_mean_squared_error: 0.5376 - dense_16_root_mean_squared_error: 0.5792 - val_loss: 2.4284 - val_dense_15_loss: 0.7678 - val_dense_16_loss: 1.6606 - val_dense_15_root_mean_squared_error: 0.8762 - val_dense_16_root_mean_squared_error: 1.2886\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6263 - dense_15_loss: 0.2911 - dense_16_loss: 0.3351 - dense_15_root_mean_squared_error: 0.5396 - dense_16_root_mean_squared_error: 0.5789 - val_loss: 1.2400 - val_dense_15_loss: 0.3405 - val_dense_16_loss: 0.8995 - val_dense_15_root_mean_squared_error: 0.5835 - val_dense_16_root_mean_squared_error: 0.9484\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6114 - dense_15_loss: 0.2860 - dense_16_loss: 0.3254 - dense_15_root_mean_squared_error: 0.5347 - dense_16_root_mean_squared_error: 0.5705 - val_loss: 0.6442 - val_dense_15_loss: 0.3134 - val_dense_16_loss: 0.3308 - val_dense_15_root_mean_squared_error: 0.5598 - val_dense_16_root_mean_squared_error: 0.5752\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6115 - dense_15_loss: 0.2868 - dense_16_loss: 0.3247 - dense_15_root_mean_squared_error: 0.5355 - dense_16_root_mean_squared_error: 0.5698 - val_loss: 0.6327 - val_dense_15_loss: 0.2896 - val_dense_16_loss: 0.3431 - val_dense_15_root_mean_squared_error: 0.5382 - val_dense_16_root_mean_squared_error: 0.5858\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6081 - dense_15_loss: 0.2848 - dense_16_loss: 0.3233 - dense_15_root_mean_squared_error: 0.5337 - dense_16_root_mean_squared_error: 0.5686 - val_loss: 0.6204 - val_dense_15_loss: 0.2903 - val_dense_16_loss: 0.3300 - val_dense_15_root_mean_squared_error: 0.5388 - val_dense_16_root_mean_squared_error: 0.5745\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6049 - dense_15_loss: 0.2842 - dense_16_loss: 0.3207 - dense_15_root_mean_squared_error: 0.5331 - dense_16_root_mean_squared_error: 0.5663 - val_loss: 0.8430 - val_dense_15_loss: 0.3660 - val_dense_16_loss: 0.4770 - val_dense_15_root_mean_squared_error: 0.6050 - val_dense_16_root_mean_squared_error: 0.6907\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6090 - dense_15_loss: 0.2865 - dense_16_loss: 0.3225 - dense_15_root_mean_squared_error: 0.5352 - dense_16_root_mean_squared_error: 0.5679 - val_loss: 0.6214 - val_dense_15_loss: 0.2850 - val_dense_16_loss: 0.3364 - val_dense_15_root_mean_squared_error: 0.5338 - val_dense_16_root_mean_squared_error: 0.5800\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6008 - dense_15_loss: 0.2817 - dense_16_loss: 0.3190 - dense_15_root_mean_squared_error: 0.5308 - dense_16_root_mean_squared_error: 0.5648 - val_loss: 0.9948 - val_dense_15_loss: 0.4204 - val_dense_16_loss: 0.5745 - val_dense_15_root_mean_squared_error: 0.6484 - val_dense_16_root_mean_squared_error: 0.7579\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5978 - dense_15_loss: 0.2805 - dense_16_loss: 0.3173 - dense_15_root_mean_squared_error: 0.5296 - dense_16_root_mean_squared_error: 0.5633 - val_loss: 0.6373 - val_dense_15_loss: 0.2862 - val_dense_16_loss: 0.3511 - val_dense_15_root_mean_squared_error: 0.5350 - val_dense_16_root_mean_squared_error: 0.5925\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6050 - dense_15_loss: 0.2838 - dense_16_loss: 0.3212 - dense_15_root_mean_squared_error: 0.5327 - dense_16_root_mean_squared_error: 0.5667 - val_loss: 0.7628 - val_dense_15_loss: 0.3228 - val_dense_16_loss: 0.4400 - val_dense_15_root_mean_squared_error: 0.5681 - val_dense_16_root_mean_squared_error: 0.6633\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5964 - dense_15_loss: 0.2802 - dense_16_loss: 0.3162 - dense_15_root_mean_squared_error: 0.5293 - dense_16_root_mean_squared_error: 0.5623 - val_loss: 0.5910 - val_dense_15_loss: 0.2734 - val_dense_16_loss: 0.3176 - val_dense_15_root_mean_squared_error: 0.5229 - val_dense_16_root_mean_squared_error: 0.5635\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6014 - dense_15_loss: 0.2822 - dense_16_loss: 0.3192 - dense_15_root_mean_squared_error: 0.5313 - dense_16_root_mean_squared_error: 0.5650 - val_loss: 1.0513 - val_dense_15_loss: 0.3974 - val_dense_16_loss: 0.6539 - val_dense_15_root_mean_squared_error: 0.6304 - val_dense_16_root_mean_squared_error: 0.8086\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5971 - dense_15_loss: 0.2794 - dense_16_loss: 0.3177 - dense_15_root_mean_squared_error: 0.5286 - dense_16_root_mean_squared_error: 0.5637 - val_loss: 0.9444 - val_dense_15_loss: 0.3564 - val_dense_16_loss: 0.5880 - val_dense_15_root_mean_squared_error: 0.5970 - val_dense_16_root_mean_squared_error: 0.7668\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5999 - dense_15_loss: 0.2807 - dense_16_loss: 0.3192 - dense_15_root_mean_squared_error: 0.5298 - dense_16_root_mean_squared_error: 0.5650 - val_loss: 1.3989 - val_dense_15_loss: 0.5738 - val_dense_16_loss: 0.8251 - val_dense_15_root_mean_squared_error: 0.7575 - val_dense_16_root_mean_squared_error: 0.9084\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5991 - dense_15_loss: 0.2814 - dense_16_loss: 0.3178 - dense_15_root_mean_squared_error: 0.5304 - dense_16_root_mean_squared_error: 0.5637 - val_loss: 0.6599 - val_dense_15_loss: 0.3033 - val_dense_16_loss: 0.3566 - val_dense_15_root_mean_squared_error: 0.5508 - val_dense_16_root_mean_squared_error: 0.5971\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5961 - dense_15_loss: 0.2796 - dense_16_loss: 0.3165 - dense_15_root_mean_squared_error: 0.5288 - dense_16_root_mean_squared_error: 0.5626 - val_loss: 0.6394 - val_dense_15_loss: 0.2959 - val_dense_16_loss: 0.3434 - val_dense_15_root_mean_squared_error: 0.5440 - val_dense_16_root_mean_squared_error: 0.5860\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5997 - dense_15_loss: 0.2811 - dense_16_loss: 0.3186 - dense_15_root_mean_squared_error: 0.5302 - dense_16_root_mean_squared_error: 0.5645 - val_loss: 0.6322 - val_dense_15_loss: 0.2881 - val_dense_16_loss: 0.3441 - val_dense_15_root_mean_squared_error: 0.5367 - val_dense_16_root_mean_squared_error: 0.5866\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5918 - dense_15_loss: 0.2777 - dense_16_loss: 0.3142 - dense_15_root_mean_squared_error: 0.5270 - dense_16_root_mean_squared_error: 0.5605 - val_loss: 0.7276 - val_dense_15_loss: 0.2958 - val_dense_16_loss: 0.4318 - val_dense_15_root_mean_squared_error: 0.5439 - val_dense_16_root_mean_squared_error: 0.6571\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5883 - dense_15_loss: 0.2761 - dense_16_loss: 0.3122 - dense_15_root_mean_squared_error: 0.5254 - dense_16_root_mean_squared_error: 0.5587 - val_loss: 0.6044 - val_dense_15_loss: 0.2821 - val_dense_16_loss: 0.3224 - val_dense_15_root_mean_squared_error: 0.5311 - val_dense_16_root_mean_squared_error: 0.5678\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5949 - dense_15_loss: 0.2794 - dense_16_loss: 0.3155 - dense_15_root_mean_squared_error: 0.5286 - dense_16_root_mean_squared_error: 0.5617 - val_loss: 0.7652 - val_dense_15_loss: 0.3119 - val_dense_16_loss: 0.4533 - val_dense_15_root_mean_squared_error: 0.5585 - val_dense_16_root_mean_squared_error: 0.6732\n"
     ]
    }
   ],
   "source": [
    "#The good news is that Keras provides a convenient TensorBoard() \n",
    "#callback that will take care of creating the log directory for you \n",
    "#(along with its parent directories if needed), and it will create \n",
    "#event files and write summaries to them during training. It will \n",
    "#measure your model’s training and validation loss and metrics \n",
    "#(in this case, the MSE and RMSE), and it will also profile your \n",
    "#neural network. It is straightforward to use:\n",
    "\n",
    "\n",
    "tensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir,\n",
    "                                               profile_batch=(100, 200))\n",
    "history = model.fit(\n",
    "        (X_train_wide, X_train_deep), (y_train, y_train), epochs=20,\n",
    "        validation_data=((X_valid_wide, X_valid_deep), (y_valid, y_valid)),\n",
    "    callbacks=[tensorboard_cb]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "d08e8482",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:19:15.653003: I tensorflow/core/profiler/lib/profiler_session.cc:101] Profiler session initializing.\n",
      "2023-03-31 14:19:15.653013: I tensorflow/core/profiler/lib/profiler_session.cc:116] Profiler session started.\n",
      "2023-03-31 14:19:15.653074: I tensorflow/core/profiler/lib/profiler_session.cc:128] Profiler session tear down.\n",
      "2023-03-31 14:19:15.874028: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126/363 [=========>....................] - ETA: 1s - loss: 0.5929 - dense_15_loss: 0.2784 - dense_16_loss: 0.3145 - dense_15_root_mean_squared_error: 0.5276 - dense_16_root_mean_squared_error: 0.5608"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:19:16.872771: I tensorflow/core/profiler/lib/profiler_session.cc:101] Profiler session initializing.\n",
      "2023-03-31 14:19:16.872783: I tensorflow/core/profiler/lib/profiler_session.cc:116] Profiler session started.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "214/363 [================>.............] - ETA: 1s - loss: 0.5914 - dense_15_loss: 0.2779 - dense_16_loss: 0.3135 - dense_15_root_mean_squared_error: 0.5272 - dense_16_root_mean_squared_error: 0.5599"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:19:17.643894: I tensorflow/core/profiler/lib/profiler_session.cc:67] Profiler session collecting data.\n",
      "2023-03-31 14:19:17.660100: I tensorflow/core/profiler/lib/profiler_session.cc:128] Profiler session tear down.\n",
      "2023-03-31 14:19:17.668485: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17\n",
      "\n",
      "2023-03-31 14:19:17.679272: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.trace.json.gz\n",
      "2023-03-31 14:19:17.694501: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17\n",
      "\n",
      "2023-03-31 14:19:17.696858: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.memory_profile.json.gz\n",
      "2023-03-31 14:19:17.698009: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17\n",
      "Dumped tool data for xplane.pb to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.xplane.pb\n",
      "Dumped tool data for overview_page.pb to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.overview_page.pb\n",
      "Dumped tool data for input_pipeline.pb to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.input_pipeline.pb\n",
      "Dumped tool data for tensorflow_stats.pb to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.tensorflow_stats.pb\n",
      "Dumped tool data for kernel_stats.pb to my_logs/run_2023_03_31_14_19_15/plugins/profile/2023_03_31_14_19_17/Rokos-MacBook-Pro-2.local.kernel_stats.pb\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - ETA: 0s - loss: 0.5900 - dense_15_loss: 0.2769 - dense_16_loss: 0.3131 - dense_15_root_mean_squared_error: 0.5262 - dense_16_root_mean_squared_error: 0.5595"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 14:19:19.059703: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363/363 [==============================] - 4s 10ms/step - loss: 0.5900 - dense_15_loss: 0.2769 - dense_16_loss: 0.3131 - dense_15_root_mean_squared_error: 0.5262 - dense_16_root_mean_squared_error: 0.5595 - val_loss: 0.7797 - val_dense_15_loss: 0.3548 - val_dense_16_loss: 0.4249 - val_dense_15_root_mean_squared_error: 0.5956 - val_dense_16_root_mean_squared_error: 0.6518\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5985 - dense_15_loss: 0.2821 - dense_16_loss: 0.3164 - dense_15_root_mean_squared_error: 0.5311 - dense_16_root_mean_squared_error: 0.5625 - val_loss: 0.9347 - val_dense_15_loss: 0.3618 - val_dense_16_loss: 0.5729 - val_dense_15_root_mean_squared_error: 0.6015 - val_dense_16_root_mean_squared_error: 0.7569\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5909 - dense_15_loss: 0.2770 - dense_16_loss: 0.3139 - dense_15_root_mean_squared_error: 0.5263 - dense_16_root_mean_squared_error: 0.5603 - val_loss: 0.6369 - val_dense_15_loss: 0.2891 - val_dense_16_loss: 0.3479 - val_dense_15_root_mean_squared_error: 0.5376 - val_dense_16_root_mean_squared_error: 0.5898\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5945 - dense_15_loss: 0.2789 - dense_16_loss: 0.3156 - dense_15_root_mean_squared_error: 0.5281 - dense_16_root_mean_squared_error: 0.5618 - val_loss: 0.6333 - val_dense_15_loss: 0.2937 - val_dense_16_loss: 0.3397 - val_dense_15_root_mean_squared_error: 0.5419 - val_dense_16_root_mean_squared_error: 0.5828\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5950 - dense_15_loss: 0.2793 - dense_16_loss: 0.3157 - dense_15_root_mean_squared_error: 0.5285 - dense_16_root_mean_squared_error: 0.5619 - val_loss: 1.2155 - val_dense_15_loss: 0.4667 - val_dense_16_loss: 0.7487 - val_dense_15_root_mean_squared_error: 0.6832 - val_dense_16_root_mean_squared_error: 0.8653\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5937 - dense_15_loss: 0.2779 - dense_16_loss: 0.3157 - dense_15_root_mean_squared_error: 0.5272 - dense_16_root_mean_squared_error: 0.5619 - val_loss: 2.1636 - val_dense_15_loss: 1.2809 - val_dense_16_loss: 0.8827 - val_dense_15_root_mean_squared_error: 1.1318 - val_dense_16_root_mean_squared_error: 0.9395\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.6022 - dense_15_loss: 0.2856 - dense_16_loss: 0.3166 - dense_15_root_mean_squared_error: 0.5344 - dense_16_root_mean_squared_error: 0.5626 - val_loss: 0.6007 - val_dense_15_loss: 0.2833 - val_dense_16_loss: 0.3175 - val_dense_15_root_mean_squared_error: 0.5322 - val_dense_16_root_mean_squared_error: 0.5635\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5874 - dense_15_loss: 0.2757 - dense_16_loss: 0.3117 - dense_15_root_mean_squared_error: 0.5251 - dense_16_root_mean_squared_error: 0.5583 - val_loss: 0.9838 - val_dense_15_loss: 0.3753 - val_dense_16_loss: 0.6085 - val_dense_15_root_mean_squared_error: 0.6126 - val_dense_16_root_mean_squared_error: 0.7801\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5841 - dense_15_loss: 0.2744 - dense_16_loss: 0.3097 - dense_15_root_mean_squared_error: 0.5238 - dense_16_root_mean_squared_error: 0.5565 - val_loss: 0.7254 - val_dense_15_loss: 0.3439 - val_dense_16_loss: 0.3815 - val_dense_15_root_mean_squared_error: 0.5864 - val_dense_16_root_mean_squared_error: 0.6176\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5887 - dense_15_loss: 0.2768 - dense_16_loss: 0.3119 - dense_15_root_mean_squared_error: 0.5261 - dense_16_root_mean_squared_error: 0.5585 - val_loss: 0.8646 - val_dense_15_loss: 0.3198 - val_dense_16_loss: 0.5448 - val_dense_15_root_mean_squared_error: 0.5655 - val_dense_16_root_mean_squared_error: 0.7381\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5811 - dense_15_loss: 0.2729 - dense_16_loss: 0.3081 - dense_15_root_mean_squared_error: 0.5224 - dense_16_root_mean_squared_error: 0.5551 - val_loss: 0.5836 - val_dense_15_loss: 0.2707 - val_dense_16_loss: 0.3129 - val_dense_15_root_mean_squared_error: 0.5203 - val_dense_16_root_mean_squared_error: 0.5593\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5853 - dense_15_loss: 0.2756 - dense_16_loss: 0.3097 - dense_15_root_mean_squared_error: 0.5250 - dense_16_root_mean_squared_error: 0.5565 - val_loss: 0.9293 - val_dense_15_loss: 0.3589 - val_dense_16_loss: 0.5704 - val_dense_15_root_mean_squared_error: 0.5991 - val_dense_16_root_mean_squared_error: 0.7552\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5850 - dense_15_loss: 0.2750 - dense_16_loss: 0.3101 - dense_15_root_mean_squared_error: 0.5244 - dense_16_root_mean_squared_error: 0.5568 - val_loss: 0.9008 - val_dense_15_loss: 0.3465 - val_dense_16_loss: 0.5543 - val_dense_15_root_mean_squared_error: 0.5886 - val_dense_16_root_mean_squared_error: 0.7445\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5883 - dense_15_loss: 0.2761 - dense_16_loss: 0.3122 - dense_15_root_mean_squared_error: 0.5254 - dense_16_root_mean_squared_error: 0.5588 - val_loss: 1.4335 - val_dense_15_loss: 0.4883 - val_dense_16_loss: 0.9452 - val_dense_15_root_mean_squared_error: 0.6988 - val_dense_16_root_mean_squared_error: 0.9722\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5830 - dense_15_loss: 0.2745 - dense_16_loss: 0.3085 - dense_15_root_mean_squared_error: 0.5239 - dense_16_root_mean_squared_error: 0.5554 - val_loss: 0.6066 - val_dense_15_loss: 0.2725 - val_dense_16_loss: 0.3340 - val_dense_15_root_mean_squared_error: 0.5221 - val_dense_16_root_mean_squared_error: 0.5780\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5794 - dense_15_loss: 0.2726 - dense_16_loss: 0.3069 - dense_15_root_mean_squared_error: 0.5221 - dense_16_root_mean_squared_error: 0.5540 - val_loss: 0.6174 - val_dense_15_loss: 0.2696 - val_dense_16_loss: 0.3477 - val_dense_15_root_mean_squared_error: 0.5192 - val_dense_16_root_mean_squared_error: 0.5897\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5857 - dense_15_loss: 0.2751 - dense_16_loss: 0.3106 - dense_15_root_mean_squared_error: 0.5245 - dense_16_root_mean_squared_error: 0.5573 - val_loss: 0.7050 - val_dense_15_loss: 0.2876 - val_dense_16_loss: 0.4174 - val_dense_15_root_mean_squared_error: 0.5363 - val_dense_16_root_mean_squared_error: 0.6461\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5771 - dense_15_loss: 0.2716 - dense_16_loss: 0.3055 - dense_15_root_mean_squared_error: 0.5212 - dense_16_root_mean_squared_error: 0.5527 - val_loss: 0.6176 - val_dense_15_loss: 0.2786 - val_dense_16_loss: 0.3390 - val_dense_15_root_mean_squared_error: 0.5278 - val_dense_16_root_mean_squared_error: 0.5822\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5756 - dense_15_loss: 0.2705 - dense_16_loss: 0.3050 - dense_15_root_mean_squared_error: 0.5201 - dense_16_root_mean_squared_error: 0.5523 - val_loss: 0.5911 - val_dense_15_loss: 0.2733 - val_dense_16_loss: 0.3178 - val_dense_15_root_mean_squared_error: 0.5228 - val_dense_16_root_mean_squared_error: 0.5637\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.5808 - dense_15_loss: 0.2738 - dense_16_loss: 0.3070 - dense_15_root_mean_squared_error: 0.5232 - dense_16_root_mean_squared_error: 0.5541 - val_loss: 0.7651 - val_dense_15_loss: 0.3035 - val_dense_16_loss: 0.4616 - val_dense_15_root_mean_squared_error: 0.5509 - val_dense_16_root_mean_squared_error: 0.6794\n"
     ]
    }
   ],
   "source": [
    "#That’s all there is to it! In this example, it will profile the network\n",
    "#between batches 100 and 200 during the first epoch. Why 100 and 200?\n",
    "#Well, it often takes a few batches for the neural network to “warm up”,\n",
    "#so you don’t want to profile too early, and profiling uses resources,\n",
    "#so it’s best not to do it for every batch.\n",
    "\n",
    "#Next, try changing the learning rate from 0.001 to 0.002, and run the \n",
    "#code again, with a new log subdirectory. You will end up with a \n",
    "#directory structure similar to this one:\n",
    "\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=2e-3)\n",
    "model.compile(loss=\"mse\", optimizer=optimizer, metrics=\n",
    "             [\"RootMeanSquaredError\"])\n",
    "\n",
    "run_logdir = get_run_logdir() #need to re-run this function for each run\n",
    "tensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir,\n",
    "                                               profile_batch=(100, 200))\n",
    "\n",
    "history = model.fit(\n",
    "        (X_train_wide, X_train_deep), (y_train, y_train), epochs=20,\n",
    "        validation_data=((X_valid_wide, X_valid_deep), (y_valid, y_valid)),\n",
    "    callbacks=[tensorboard_cb]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "78a9066e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('my_logs/run_2023_03_31_10_36_18')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_logdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d5808992",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-b9c82d750b93faa0\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-b9c82d750b93faa0\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#There’s one directory per run, each containing one subdirectory for \n",
    "#training logs and one for validation logs. Both contain event files, \n",
    "#and the training logs also include profiling traces.\n",
    "\n",
    "#Now that you have the event files ready, it’s time to start the \n",
    "#TensorBoard server. This can be done directly within Jupyter or \n",
    "#Colab using the Jupyter extension for TensorBoard, which gets \n",
    "#installed along with the TensorBoard library. This extension is \n",
    "#preinstalled in Colab. The following code loads the Jupyter \n",
    "#extension for TensorBoard, and the second line starts a TensorBoard \n",
    "#server for the my_logs directory, connects to this server and displays\n",
    "#the user interface directly inside of Jupyter. The server, listens on \n",
    "#the first available TCP port greater than or equal to 6006 (or you can \n",
    "#set the port you want using the --port option).\n",
    "\n",
    "#TIP\n",
    "#If you’re running everything on your own machine, it’s possible to\n",
    "#start TensorBoard by executing tensorboard --logdir=./my_logs in a\n",
    "#terminal. You must first activate the Conda environment in which you\n",
    "#installed TensorBoard, and go to the handson-ml3 directory. Once the\n",
    "#server is started, visit http://localhost:6006.\n",
    "\n",
    "%load_ext tensorboard\n",
    "%tensorboard --logdir=./my_logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "fcb8ba39",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now you should see TensorBoard’s user interface. Click the SCALARS tab to view the learning \n",
    "#curves (see Figure 10-16). At the bottom left, select the logs you want to visualize (e.g., \n",
    "#the training logs from the first and second run), and click the epoch_loss scalar. Notice \n",
    "#that the training loss went down nicely during both runs, but in the second run it went down \n",
    "#a bit faster thanks to the higher learning rate.\n",
    "\n",
    "\n",
    "#You can also visualize the whole computation graph in the GRAPHS tab, the learned weights \n",
    "#projected to 3D in the PROJECTOR tab, and the profiling traces in the PROFILE tab. The \n",
    "#TensorBoard() callback has options to log extra data too (see the documentation for more details). \n",
    "#You can click the refresh button (⟳) at the top right to make TensorBoard refresh data, and you can \n",
    "#click the settings button (⚙) to activate auto- refresh and specify the refresh interval.\n",
    "\n",
    "\n",
    "\n",
    "#Additionally, TensorFlow offers a lower-level API in the tf.summary package. The following code \n",
    "#creates a SummaryWriter using the create_file_writer() function, and it uses this writer as a \n",
    "#Python context to log scalars, histograms, images, audio, and text, all of which can then be \n",
    "#visualized using TensorBoard:\n",
    "\n",
    "\n",
    "test_logdir = get_run_logdir()\n",
    "writer = tf.summary.create_file_writer(str(test_logdir))\n",
    "\n",
    "with writer.as_default():\n",
    "    for step in range(1, 1000 + 1):\n",
    "        tf.summary.scalar(\"my_scalar\", np.sin(step / 10), step=step)\n",
    "        \n",
    "        data = (np.random.randn(100) + 2) * step / 100 # gets larger\n",
    "        tf.summary.histogram(\"my_hist\", data, buckets=50, step=step)\n",
    "        \n",
    "        images = np.random.rand(2, 32, 32, 3) * step / 1000 # gets brighter\n",
    "        tf.summary.image(\"my_images\", images, step=step)\n",
    "        \n",
    "        texts = [\"The step is \" + str(step), \"Its square is \" + str(step ** 2)]\n",
    "        \n",
    "        tf.summary.text(\"my_text\", texts, step=step)\n",
    "        sine_wave = tf.math.sin(tf.range(12000) / 48000 * 2 * np.pi * step)\n",
    "        audio = tf.reshape(tf.cast(sine_wave, tf.float32), [1, -1, 1])\n",
    "        tf.summary.audio(\"my_audio\", audio, sample_rate=48000, step=step)\n",
    "\n",
    "        \n",
    "        \n",
    "#If you run this code and click the refresh button in TensorBoard, you will see several\n",
    "#tabs appear: IMAGES, AUDIO, DISTRIBUTIONS, HISTOGRAMS, and TEXT. Try clicking the IMAGES\n",
    "#tab, and use the slider above each image to view the images at different time steps. \n",
    "#Similarly, go to the AUDIO tab and try listening to the audio at different time steps.\n",
    "#As you can see, TensorBoard is a useful tool even beyond TensorFlow or deep learning.\n",
    "\n",
    "\n",
    "\n",
    "#Let’s summarize what you’ve learned so far in this chapter: you now know where neural \n",
    "#nets came from, what an MLP is and how you can use it for classification and regression,\n",
    "#how to use Keras’s sequential API to build MLPs, and how to use the functional API or \n",
    "#the subclassing API to build more complex model architectures (including Wide & Deep \n",
    "#models, as well as models with multiple inputs and outputs). You also learned how \n",
    "#to save and restore a model and how to use callbacks for checkpointing, early stopping, \n",
    "#and more. Finally, you learned how to use TensorBoard for visualization. You can already\n",
    "#go ahead and use neural networks to tackle many problems! However, you may wonder how\n",
    "#to choose the number of hidden layers, the number of neurons in the network, and all\n",
    "#the other hyperparameters. Let’s look at this now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "827ae1e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fine-Tuning Neural Network Hyperparameters\n",
    "\n",
    "#The flexibility of neural networks is also one of their main drawbacks:\n",
    "#there are many hyperparameters to tweak. Not only can you use any\n",
    "#imaginable network architecture, but even in a basic MLP you can \n",
    "#change the number of layers, the number of neurons and the type of\n",
    "#activation function to use in each layer, the weight initialization\n",
    "#logic, the type of optimizer to use, its learning rate, the batch size,\n",
    "#and more. How do you know what combination of hyperparameters is the\n",
    "#best for your task?\n",
    "\n",
    "#One option is to convert your Keras model to a Scikit-Learn estimator,\n",
    "#and then use GridSearchCV or RandomizedSearchCV to fine-tune the\n",
    "#hyperparameters, as you did in Chapter 2. For this, you can use\n",
    "#the KerasRegressor and KerasClassifier wrapper classes from the\n",
    "#SciKeras library (see https://github.com/adriangb/scikeras for\n",
    "#more details). However, there’s a better way: you can use the\n",
    "#Keras Tuner library, which is a hyperparameter tuning library\n",
    "#for Keras models. It offers several tuning strategies, it’s highly\n",
    "#customizable, and it has excellent integration with TensorBoard.\n",
    "#Let’s see how to use it.\n",
    "\n",
    "#import keras_tuner, usually as kt, then write a function that builds,\n",
    "#compiles, and returns a Keras model. The function must take a \n",
    "#kt.HyperParameters object as an argument, which it can use to define\n",
    "#hyperparameters (integers, floats, strings, etc.) along with their\n",
    "#range of possible values, and these hyperparameters may be used to\n",
    "#build and compile the model. For example, the following function\n",
    "#builds and compiles an MLP to classify Fashion MNIST images,\n",
    "#using hyperparameters such as the number of hidden layers (n_hidden),\n",
    "#the number of neurons per layer (n_neurons), the learning rate\n",
    "#(learning_rate), and the type of optimizer to use (optimizer):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "b7a71c43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#pip install -q -U keras-tuner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "9c7de181",
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n",
    "\n",
    "def build_model(hp):\n",
    "    n_hidden = hp.Int(\"n_hidden\", min_value=0, max_value=8, default=2)\n",
    "    n_neurons = hp.Int(\"n_neurons\", min_value=16, max_value=256)\n",
    "    learning_rate = hp.Float(\"learning_rate\", min_value=1e-4, \n",
    "                             max_value=1e-2, sampling=\"log\")\n",
    "    optimizer = hp.Choice(\"optimizer\", values=[\"sgd\", \"adam\"])\n",
    "    if optimizer == \"sgd\":\n",
    "        optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "    else:\n",
    "        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "    \n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(tf.keras.layers.Flatten())\n",
    "    for _ in range(n_hidden):\n",
    "        model.add(tf.keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
    "    model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n",
    "    model.compile(loss=\"sparse_categorical_crossentropy\", \n",
    "                 optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e7768efa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 5 Complete [00h 00m 27s]\n",
      "val_accuracy: 0.004392764996737242\n",
      "\n",
      "Best val_accuracy So Far: 0.004909561015665531\n",
      "Total elapsed time: 00h 02m 10s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "#The first part of the function defines the hyperparameters. For example,\n",
    "#hp.Int(\"n_hidden\", min_value=0, max_value=8, default=2) checks whether \n",
    "#a hyperparameter named \"n_hidden\" is already present in the \n",
    "#HyperParameters object hp, and if so it returns its value. If not,\n",
    "#then it registers a new integer hyperparameter named \"n_hidden\", \n",
    "#whose possible values range from 0 to 8 (inclusive), and it returns\n",
    "#the default value, which is 2 in this case (when default is not set,\n",
    "#then min_value is returned). The \"n_neurons\" hyperparameter is \n",
    "#registered in a similar way. The \"learning_rate\" hyperparameter is\n",
    "#registered as a float ranging from 10–4 to 10–2, and since \n",
    "#sampling=\"log\", learning rates of all scales will be sampled \n",
    "#equally. Lastly, the optimizer hyperparameter is registered with \n",
    "#two possible values: \"sgd\" or \"adam\" (the default value is the first \n",
    "#one, which is \"sgd\" in this case). Depending on the value of optimizer,\n",
    "#we create an SGD optimizer or an Adam optimizer with the given \n",
    "#learning rate.\n",
    "\n",
    "#The second part of the function just builds the model using the \n",
    "#hyperparameter values. It creates a Sequential model starting \n",
    "#with a Flatten layer, followed by the requested number of hidden \n",
    "#layers (as determined by the n_hidden hyperparameter) using the \n",
    "#ReLU activation function, and an output layer with 10 neurons \n",
    "#(one per class) using the softmax activation function. Lastly, \n",
    "#the function compiles the model and returns it.\n",
    "\n",
    "#Now if you want to do a basic random search, you can create a \n",
    "#kt.RandomSearch tuner, passing the build_model function to the\n",
    "#constructor, and call the tuner’s search() method:\n",
    "\n",
    "\n",
    "random_search_tuner = kt.RandomSearch(\n",
    "        build_model, objective=\"val_accuracy\", max_trials=5,\n",
    "        overwrite=True, directory=\"my_fashion_mnist_2\", \n",
    "        project_name=\"my_rnd_search\", seed=42)\n",
    "\n",
    "random_search_tuner.search(X_train, y_train,\n",
    "                          epochs=10, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "04601431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oracle.json  \u001b[34mtrial_1\u001b[m\u001b[m/     \u001b[34mtrial_3\u001b[m\u001b[m/     tuner0.json\r\n",
      "\u001b[34mtrial_0\u001b[m\u001b[m/     \u001b[34mtrial_2\u001b[m\u001b[m/     \u001b[34mtrial_4\u001b[m\u001b[m/\r\n"
     ]
    }
   ],
   "source": [
    "ls my_fashion_mnist_2/my_rnd_search/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "22ff8e19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.engine.sequential.Sequential at 0x29f45e6e0>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The RandomSearch tuner first calls build_model() once with an \n",
    "#empty Hyperparameters object, just to gather all the hyperparameter\n",
    "#specifications. Then, in this example, it runs 5 trials; for each \n",
    "#trial it builds a model using hyperparameters sampled randomly\n",
    "#within their respective ranges, then it trains that model for\n",
    "#10 epochs and saves it to a subdirectory of the \n",
    "#my_fashion_mnist/my_rnd_search directory. Since overwrite=True,\n",
    "#the my_rnd_search directory is deleted before training starts.\n",
    "#If you run this code a second time but with overwrite=False and\n",
    "#max_trials=10, the tuner will continue tuning where it left off,\n",
    "#running 5 more trials: this means you don’t have to run all the\n",
    "#trials in one shot. Lastly, since objective is set to \"val_accuracy\",\n",
    "#the tuner prefers models with a higher validation accuracy, so once\n",
    "#the tuner has finished searching, you can get the best models like this:\n",
    "\n",
    "\n",
    "top3_models = random_search_tuner.get_best_models(num_models=3)\n",
    "best_model = top3_models[0]\n",
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "5d5f620b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_hidden': 5,\n",
       " 'n_neurons': 25,\n",
       " 'learning_rate': 0.0006562536901904111,\n",
       " 'optimizer': 'adam'}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#You can also call get_best_hyperparameters() to get the\n",
    "#kt.HyperParameters of the best models:\n",
    "\n",
    "top3_params = random_search_tuner.get_best_hyperparameters(num_trials=3)\n",
    "top3_params[0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "b5f61bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial summary\n",
      "Hyperparameters:\n",
      "n_hidden: 5\n",
      "n_neurons: 25\n",
      "learning_rate: 0.0006562536901904111\n",
      "optimizer: adam\n",
      "Score: 0.004909561015665531\n"
     ]
    }
   ],
   "source": [
    "best_trial = random_search_tuner.oracle.get_best_trials(num_trials=1)[0]\n",
    "best_trial.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "677b67dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.004909561015665531"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This shows the best hyperparameters (like earlier), as well as the \n",
    "#validation accuracy. You can also access all the metrics directly:\n",
    "\n",
    "best_trial.metrics.get_last_value(\"val_accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "0d5e1863",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "  9/484 [..............................] - ETA: 3s - loss: 1.3973 - accuracy: 0.0000e+00  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:46:18.714931: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "484/484 [==============================] - 3s 6ms/step - loss: 1.3906 - accuracy: 0.0029\n",
      "Epoch 2/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.3104 - accuracy: 0.0030\n",
      "Epoch 3/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.2562 - accuracy: 0.0030\n",
      "Epoch 4/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.2403 - accuracy: 0.0024\n",
      "Epoch 5/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.2127 - accuracy: 0.0028\n",
      "Epoch 6/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.1994 - accuracy: 0.0025\n",
      "Epoch 7/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.1811 - accuracy: 0.0026\n",
      "Epoch 8/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.1979 - accuracy: 0.0025\n",
      "Epoch 9/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.1715 - accuracy: 0.0026\n",
      "Epoch 10/10\n",
      "484/484 [==============================] - 3s 6ms/step - loss: 1.1796 - accuracy: 0.0024\n",
      " 26/162 [===>..........................] - ETA: 0s - loss: 1.1637 - accuracy: 0.0012   "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-31 10:46:47.700190: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 1s 4ms/step - loss: 1.1654 - accuracy: 0.0014\n"
     ]
    }
   ],
   "source": [
    "#If you are happy with the best model’s performance, you may continue \n",
    "#training it for a few epochs on the full training set (X_train_full and\n",
    "#y_train_full), then evaluate it on the test set, and \n",
    "#deploy it to production (see Chapter 19):\n",
    "\n",
    "best_model.fit(X_train_full, y_train_full, epochs=10)\n",
    "test_loss, test_accuracy = best_model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6326aba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#In some cases, you may want to fine-tune data preprocessing hyperparameters, or model.fit()\n",
    "#arguments, such as the batch size. For this, you must use a slightly different technique:\n",
    "#instead of writing a build_model() function, you must subclass the kt.HyperModel class\n",
    "#and define two methods, build() and fit(). The build() method does the exact same thing\n",
    "#as the build_model() function. The fit() method takes a HyperParameters object and a\n",
    "#compiled model as an argument, as well as all the model.fit() arguments, and fits\n",
    "#the model and returns the History object. Crucially, the fit() method may use\n",
    "#hyperparameters to decide how to preprocess the data, tweak the batch size, and more.\n",
    "#For example, the following class builds the same model as before, with the same \n",
    "#hyperparameters, but it also uses a Boolean \"normalize\" hyperparameter to control\n",
    "#whether or not to standardize the training data before fitting the model:\n",
    "\n",
    "class MyClassificationHyperModel(kt.HyperModel):\n",
    "    def build(self, hp):\n",
    "        return build_model(hp)\n",
    "    \n",
    "    def fit(self, hp, model, X, y, **kwargs):\n",
    "        if hp.Boolean(\"normalize\"):\n",
    "            norm_layer = tf.keras.layers.Normalization()\n",
    "            X = norm_layer(X)\n",
    "        return model.fit(X, y, **kwargs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "f3dc5966",
   "metadata": {},
   "outputs": [],
   "source": [
    "#You can then pass an instance of this class to the tuner of your choice, instead of\n",
    "#passing the build_model function. For example, let’s build a kt.Hyperband tuner \n",
    "#based on a MyClassificationHyperModel instance:\n",
    "\n",
    "\n",
    "hyperband_tuner = kt.Hyperband(\n",
    "        MyClassificationHyperModel(), objective=\"val_accuracy\", seed=42,\n",
    "        max_epochs=10, factor=3, hyperband_iterations=2, overwrite=True,\n",
    "        directory=\"my_fashion_mnist\", project_name=\"hyperband\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "f1101198",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 72 Complete [00h 00m 11s]\n",
      "val_accuracy: 0.004392764996737242\n",
      "\n",
      "Best val_accuracy So Far: 0.0051679592579603195\n",
      "Total elapsed time: 03h 35m 09s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "#This tuner is similar to the HalvingRandomSearchCV class we discussed in Chapter 2: it\n",
    "#starts by training many different models for few epochs, then it eliminates the worst\n",
    "#models and keeps only the top 1 / factor models (i.e., the top third in this case),\n",
    "#repeating this selection process until a single model is left.19 The max_epochs argument\n",
    "#controls the max number of epochs that the best model will be trained for. The whole\n",
    "#process is repeated twice in this case (hyperband_iterations=2). The total number of\n",
    "#training epochs across all models for each hyperband iteration is about \n",
    "#max_epochs * (log(max_epochs) / log(factor)) ** 2, so it’s about 44 epochs\n",
    "#in this example. The other arguments are the same as for kt.RandomSearch.\n",
    "\n",
    "#Let’s run the Hyperband tuner now. We’ll use the TensorBoard callback, this time pointing\n",
    "#to the root log directory (the tuner will take care of using a different subdirectory\n",
    "#for each trial), as well as an EarlyStopping callback:\n",
    "\n",
    "root_logdir = Path(hyperband_tuner.project_dir) / \"tensorboard\"\n",
    "tensorboard_cb = tf.keras.callbacks.TensorBoard(root_logdir)\n",
    "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=2)\n",
    "hyperband_tuner.search(X_train, y_train, epochs=10, \n",
    "                      validation_data=(X_valid, y_valid),\n",
    "                      callbacks=[early_stopping_cb, tensorboard_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "a53b952e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tensorboard extension is already loaded. To reload it, use:\n",
      "  %reload_ext tensorboard\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-6d10880b7f324083\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-6d10880b7f324083\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6008;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Now if you open TensorBoard, pointing --logdir to the my_fashion_mnist/hyperband/tensorboard directory, \n",
    "#you will see all the trial results as they unfold. Make sure to visit the HPARAMS tab: it contains a \n",
    "#summary of all the hyperparameter combinations that were tried, along with the corresponding metrics.\n",
    "#Notice that there are three tabs inside the HPARAMS tab: a table view, a parallel coordinates view, and\n",
    "#a scatterplot matrix view. In the lower part of the left panel, uncheck all metrics except for\n",
    "#validation.epoch_accuracy: this will make the graphs clearer. In the parallel coordinates view, try selecting\n",
    "#a range of high values in the validation.epoch_accuracy column: this will filter only the hyperparameter \n",
    "#combinations that reached a good performance. Click one of the hyperparameter combinations, and the \n",
    "#corresponding learning curves will appear at the bottom of the page. Take some time to go through each tab; \n",
    "#this will help you understand the effect of each hyperparameter on performance, as well as the interactions\n",
    "#between the hyperparameters.\n",
    "\n",
    "%load_ext tensorboard\n",
    "%tensorboard --logdir=my_fashion_mnist/hyperband/tensorboard\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92219980",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperband is smarter than pure random search in the way it allocates resources, but at its core it \n",
    "#still explores the hyperparameter space randomly; it’s fast, but coarse. However, Keras Tuner also \n",
    "#includes a kt.BayesianOptimization tuner: this algorithm gradually learns which regions of the hyperparameter\n",
    "#space are most promising by fitting a probabilistic model called a Gaussian process. This allows it to \n",
    "#gradually zoom in on the best hyperparameters. The downside is that the algorithm has its own hyperparameters:\n",
    "#alpha represents the level of noise you expect in the performance measures across trials (it defaults to 10–4),\n",
    "#and beta specifies how much you want the algorithm to explore, instead of simply exploiting the known good \n",
    "#regions of hyperparameter space (it defaults to 2.6). Other than that, this tuner can be used just like the\n",
    "#previous ones:\n",
    "\n",
    "\n",
    "\n",
    "bayesian_opt_tuner = kt.BayesianOptimization(\n",
    "                MyClassificationHyperModel(), objective=\"val_accuracy\", seed=42,\n",
    "    max_trials=10, alpha=1e-4, beta=2.6, overwrite=True, directory=\"my_fashion_mnist\",\n",
    "    project_name=\"bayesian_opt\")\n",
    "\n",
    "\n",
    "bayesian_opt_tuner.search([...])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa0898da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameter tuning is still an active area of research, and many other approaches are being explored. \n",
    "#For example, check out DeepMind’s excellent 2017 paper,20 where the authors used an evolutionary algorithm \n",
    "#to jointly optimize a population of models and their hyperparameters. Google has also used an evolutionary \n",
    "#approach, not just to search for hyperparameters but also to explore all sorts of model architectures: it \n",
    "#powers their AutoML service on Google Vertex AI (see Chapter 19). The term AutoML refers to any system that\n",
    "#takes care of a large part of the ML workflow. Evolutionary algorithms have even been used successfully to\n",
    "#train individual neural networks, replacing the ubiquitous gradient descent!\n",
    "\n",
    "\n",
    "#For an example, see the 2017 post by Uber where the authors introduce their Deep Neuroevolution technique.\n",
    "#But despite all this exciting progress and all these tools and services, it still helps to have an idea\n",
    "#of what values are reasonable for each hyperparameter so that you can build a quick prototype and restrict\n",
    "#the search space. The following sections provide guidelines for choosing the number of hidden layers and\n",
    "#neurons in an MLP and for selecting good values for some of the main hyperparameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e39319b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Number of Hidden Layers\n",
    "\n",
    "\n",
    "#For many problems, you can begin with a single hidden layer and get reasonable results. An MLP with just one \n",
    "#hidden layer can theoretically model even the most complex functions, provided it has enough neurons. But\n",
    "#for complex problems, deep networks have a much higher parameter efficiency than shallow ones: they can model\n",
    "#complex functions using exponentially fewer neurons than shallow nets, allowing them to reach much better \n",
    "#performance with the same amount of training data.\n",
    "\n",
    "#To understand why, suppose you are asked to draw a forest using some drawing software, but you are \n",
    "#forbidden to copy and paste anything. It would take an enormous amount of time: you would have to draw\n",
    "#each tree individually, branch by branch, leaf by leaf. If you could instead draw one leaf, copy and paste it \n",
    "#to draw a branch, then copy and paste that branch to create a tree, and finally copy and paste this tree to \n",
    "#make a forest, you would be finished in no time. Real-world data is often structured in such a hierarchical \n",
    "#way, and deep neural networks automatically take advantage of this fact: lower hidden layers model low-level\n",
    "#structures (e.g., line segments of various shapes and orientations), intermediate hidden layers combine these\n",
    "#low-level structures to model intermediate-level structures (e.g., squares, circles), and the highest hidden\n",
    "#layers and the output layer combine these intermediate structures to model high-level structures (e.g., faces).\n",
    "\n",
    "#Not only does this hierarchical architecture help DNNs converge faster to a good solution, but it also improves\n",
    "#their ability to generalize to new datasets. For example, if you have already trained a model to recognize faces\n",
    "#in pictures and you now want to train a new neural network to recognize hairstyles, you can kickstart the\n",
    "#training by reusing the lower layers of the first network. Instead of randomly initializing the weights\n",
    "#and biases of the first few layers of the new neural network, you can initialize them to the values of\n",
    "#the weights and biases of the lower layers of the first network. This way the network will not have to\n",
    "#learn from scratch all the low-level structures that occur in most pictures; it will only have to learn\n",
    "#the higher-level structures (e.g., hairstyles). This is called transfer learning.\n",
    "\n",
    "#In summary, for many problems you can start with just one or two hidden layers and the neural network will\n",
    "#work just fine. For instance, you can easily reach above 97% accuracy on the MNIST dataset using just one\n",
    "#hidden layer with a few hundred neurons, and above 98% accuracy using two hidden layers with the same \n",
    "#total number of neurons, in roughly the same amount of training time. For more complex problems, you can\n",
    "#ramp up the number of hidden layers until you start overfitting the training set. Very complex tasks,\n",
    "#such as large image classification or speech recognition, typically require networks with dozens of\n",
    "#layers (or even hundreds, but not fully connected ones, as you will see in Chapter 14), and they\n",
    "#need a huge amount of training data. You will rarely have to train such networks from scratch: it\n",
    "#is much more common to reuse parts of a pretrained state-of-the-art network that performs a similar\n",
    "#task. Training will then be a lot faster and require much less data (we will discuss this in Chapter 11).    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f036dc36",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Number of Neurons per Hidden Layer\n",
    "\n",
    "#The number of neurons in the input and output layers is determined by the type of input and output your\n",
    "#task requires. For example, the MNIST task requires 28 × 28 = 784 inputs and 10 output neurons.\n",
    "#As for the hidden layers, it used to be common to size them to form a pyramid, with fewer and fewer\n",
    "#neurons at each layer—the rationale being that many low-level features can coalesce into far fewer\n",
    "#high-level features. A typical neural network for MNIST might have 3 hidden layers, the first with \n",
    "#300 neurons, the second with 200, and the third with 100. However, this practice has been largely \n",
    "#abandoned because it seems that using the same number of neurons in all hidden layers performs just\n",
    "#as well in most cases, or even better; plus, there is only one hyperparameter to tune, instead of\n",
    "#one per layer. That said, depending on the dataset, it can sometimes help to make the first hidden\n",
    "#layer bigger than the others.\n",
    "\n",
    "\n",
    "#Just like the number of layers, you can try increasing the number of neurons gradually until the network\n",
    "#starts overfitting. Alternatively, you can try building a model with slightly more layers and neurons\n",
    "#than you actually need, then use early stopping and other regularization techniques to prevent it from\n",
    "#overfitting too much. Vincent Vanhoucke, a scientist at Google, has dubbed this the “stretch pants” approach:\n",
    "#instead of wasting time looking for pants that perfectly match your size, just use large stretch pants that\n",
    "#will shrink down to the right size. With this approach, you avoid bottleneck layers that could ruin your model.\n",
    "#Indeed, if a layer has too few neurons, it will not have enough representational power to preserve all the\n",
    "#useful information from the inputs (e.g., a layer with two neurons can only output 2D data, so if it gets\n",
    "#3D data as input, some information will be lost). No matter how big and powerful the rest of the network is,\n",
    "#that information will never be recovered.\n",
    "\n",
    "#TIP\n",
    "#In general you will get more bang for your buck by increasing the number of layers instead of the number\n",
    "#of neurons per layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ddbd0b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Learning Rate, Batch Size, and Other Hyperparameters\n",
    "\n",
    "#The number of hidden layers and neurons are not the only hyperparameters you can tweak in an MLP. \n",
    "#Here are some of the most important ones, as well as tips on how to set them:\n",
    "\n",
    "\n",
    "#Learning rate\n",
    "#The learning rate is arguably the most important hyperparameter. In general, the optimal learning rate is \n",
    "#about half of the maximum learning rate (i.e., the learning rate above which the training algorithm diverges,\n",
    "#as we saw in Chapter 4). One way to find a good learning rate is to train the model for a few hundred iterations,\n",
    "#starting with a very low learning rate (e.g., 10–5) and gradually increasing it up to a very large value (e.g., 10).\n",
    "#This is done by multiplying the learning rate by a constant factor at each iteration (e.g., by (10 / 10-5)1 / 500\n",
    "#to go from 10–5 to 10 in 500 iterations). If you plot the loss as a function of the learning rate (using a log\n",
    "#scale for the learning rate), you should see it dropping at first. But after a while, the learning rate will\n",
    "#be too large, so the loss will shoot back up: the optimal learning rate will be a bit lower than the point\n",
    "#at which the loss starts to climb (typically about 10 times lower than the turning point). You can then\n",
    "#reinitialize your model and train it normally using this good learning rate. We will look at more learning\n",
    "#rate optimization techniques in Chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1beac1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Optimizer\n",
    "#Choosing a better optimizer than plain old mini-batch gradient descent (and tuning its hyperparameters)\n",
    "#is also quite important. We will examine several advanced optimizers in Chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "431d760e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Batch size\n",
    "#The batch size can have a significant impact on your model’s performance and training time. The main benefit\n",
    "#of using large batch sizes is that hardware accelerators like GPUs can process them efficiently (see Chapter 19),\n",
    "#so the training algorithm will see more instances per second. Therefore, many researchers and practitioners\n",
    "#recommend using the largest batch size that can fit in GPU RAM. There’s a catch, though: in practice, large\n",
    "#batch sizes often lead to training instabilities, especially at the beginning of training, and the resulting\n",
    "#model may not generalize as well as a model trained with a small batch size. In April 2018, Yann LeCun even\n",
    "#tweeted “Friends don’t let friends use mini-batches larger than 32”, citing a 2018 paper by Dominic Masters \n",
    "#and Carlo Luschi which concluded that using small batches (from 2 to 32) was preferable because small batches\n",
    "#led to better models in less training time. Other research points in the opposite direction, however. For example,\n",
    "#in 2017, papers by Elad Hoffer et al.22 and Priya Goyal et al.23 showed that it was possible to use very\n",
    "#large batch sizes (up to 8,192) along with various techniques such as warming up the learning rate (i.e.,\n",
    "#starting training with a small learning rate, then ramping it up, as discussed in Chapter 11) and to obtain\n",
    "#very short training times, without any generalization gap. So, one strategy is to try to using a large batch\n",
    "#size, with learning rate warmup, and if training is unstable or the final performance is disappointing, then\n",
    "#try using a small batch size instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7d6869b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Activation function\n",
    "#We discussed how to choose the activation function earlier in this chapter: in general, the ReLU activation\n",
    "#function will be a good default for all hidden layers, but for the output layer it really depends on your task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aac2b38e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Number of iterations\n",
    "#In most cases, the number of training iterations does not actually need to be tweaked: just use early\n",
    "#stopping instead.\n",
    "#TIP\n",
    "#The optimal learning rate depends on the other hyperparameters—especially the batch size—so if you modify\n",
    "#any hyperparameter, make sure to update the learning rate as well.\n",
    "\n",
    "\n",
    "#For more best practices regarding tuning neural network hyperparameters, check out the excellent \n",
    "#2018 paper24 by Leslie Smith.\n",
    "\n",
    "\n",
    "#This concludes our introduction to artificial neural networks and their implementation with Keras.\n",
    "#In the next few chapters, we will discuss techniques to train very deep nets. We will also explore\n",
    "#how to customize models using TensorFlow’s lower-level API and how to load and preprocess data\n",
    "#efficiently using the tf.data API. And we will dive into other popular neural network architectures:\n",
    "#convolutionalneural networks for image processing, recurrent neural networks and transformers for\n",
    "#sequential data and text, autoencoders for representation learning, and generative adversarial\n",
    "#networks to model and generate data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "981bfa54",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e692aca1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "835b79fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a61e87a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f3e81ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23236b60",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a14f8208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22f0898d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bad4f29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe03b2a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a6ef51f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd2a71d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "984b6806",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}