From 42a4d5f63bf980b8108407ca28f9825ef279251b Mon Sep 17 00:00:00 2001 From: Lopes Date: Fri, 10 Apr 2026 11:20:20 +0200 Subject: [PATCH 1/6] Edit the notebook according to the suggestions given by Edoardo --- 30_introduction_data_exploration.ipynb | 60 ++++++++++++++++++++------ 31_image_classification.ipynb | 40 +++++++++++++---- data/data_exploration/bubbly.py | 32 ++++++++------ 3 files changed, 98 insertions(+), 34 deletions(-) diff --git a/30_introduction_data_exploration.ipynb b/30_introduction_data_exploration.ipynb index 01ec0236..46626058 100644 --- a/30_introduction_data_exploration.ipynb +++ b/30_introduction_data_exploration.ipynb @@ -334,9 +334,7 @@ " Returns:\n", " pd.DataFrame: A DataFrame containing the happiness data.\n", " \"\"\"\n", - " # Your code starts here\n", - " return\n", - " # Your code ends here" + " return" ] }, { @@ -593,9 +591,33 @@ " Returns:\n", " - Cleaned DataFrame with missing values filled\n", " \"\"\"\n", - " # Your code starts here\n", - " return \n", - " # Your code ends here" + " min_year = happiness_df[\"year\"].min()\n", + " max_year = happiness_df[\"year\"].max()\n", + " possible_years = np.arange(min_year, max_year + 1)\n", + " possible_countries = happiness_df[\"Country name\"].unique()\n", + "\n", + " # Create entries for each country for every year\n", + " all_years_countries = pd.DataFrame(\n", + " [(country, year) for country in possible_countries for year in possible_years],\n", + " columns=[\"Country name\", \"year\"],\n", + " )\n", + " # Extend the happiness_df to include all years for each country\n", + "\n", + " complete_happiness_df = all_years_countries.merge(\n", + " happiness_df, on=[\"Country name\", \"year\"], how=\"left\"\n", + " )\n", + "\n", + " # Set initial values to 1:\n", + " year_2005 = complete_happiness_df[\"year\"] == 2005\n", + " complete_happiness_df.loc[year_2005] = complete_happiness_df.loc[year_2005].fillna(\n", + " 1\n", + " )\n", + " # Apply forward fill for any remaining NaNs (make sure sorting is in the right order)\n", + " complete_happiness_df = complete_happiness_df.sort_values(\n", + " by=[\"Country name\", \"year\"]\n", + " ).ffill()\n", + "\n", + " return complete_happiness_df" ] }, { @@ -707,9 +729,9 @@ " Returns:\n", " - DataFrame with the regional indicator added\n", " \"\"\"\n", - " # Your code starts here\n", - " return\n", - " # Your code ends here" + " final_happiness_df = pd.merge(cleaned_happiness_df, region_df, on='Country name', how='left')\n", + " final_happiness_df['Regional indicator'] = final_happiness_df['Regional indicator'].fillna('Unknown')\n", + " return final_happiness_df" ] }, { @@ -1213,9 +1235,14 @@ " Returns:\n", " - Dictionary containing the trace and frame information\n", " \"\"\"\n", - " # Your code starts here\n", - " return\n", - " # Your code ends here" + " \"\"\"Make a frame for a given year with bubble size\"\"\"\n", + "\n", + " def trace_by_category(dataset, year, x_column, y_column, description_column, category_column, category, bubble_size_column):\n", + " \"\"\"Make a trace for a given year with bubble size\"\"\"\n", + " trace = {'x': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), x_column]), 'y': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), y_column]), 'mode': 'markers', 'text': list(dataset.loc[dataset['year'] == year, description_column]), 'marker': {'size': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), bubble_size_column]), 'sizemode': 'area', 'sizeref': 1}, 'type': 'scatter', 'name': category}\n", + " return trace\n", + " frame = {'data': [trace_by_category(dataset, year, x_column, y_column, description_column, category_column, category, bubble_size_column) for category in dataset[category_column].unique()], 'name': str(year)}\n", + " return frame" ] }, { @@ -1290,6 +1317,13 @@ " x_logscale=True, scale_bubble=3, height=650)\n", "iplot(figure, config={'scrollZoom': True})\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1308,7 +1342,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.10" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/31_image_classification.ipynb b/31_image_classification.ipynb index d7e8b86b..d9cce389 100644 --- a/31_image_classification.ipynb +++ b/31_image_classification.ipynb @@ -155,12 +155,12 @@ "return len(self.images)\n", "```\n", "\n", - "Complete function ```__getitem__``` - this method is needed to lety the generator know what to do to samples when calling them:\n", + "Complete function ```__getitem__``` - this method is needed to let the generator know what to do to samples when calling them:\n", "```python\n", "image = self.images[idx]\n", "label = self.labels[idx]\n", "\n", - "# Ensure the image is in the shape (H, W, C) for Albumentations library (library used for image augmentation)\n", + "# Pytorch expects images with the shape (Channels, Height, Width). However, Albumentations library (library used for image augmentation) requires them with the shape (Height, Width, Channels). Therefore, we need to use transpose to make them compatible with the library. Later in the notebook the images are converted back to the original shape by using A.ToTensorV2().\n", "image = np.transpose(image, (1, 2, 0))\n", "\n", "# Apply transformations on the images\n", @@ -549,10 +549,11 @@ "outputs": [], "source": [ "import numpy as np\n", + "import torch\n", "\n", "class Trainer():\n", " def __init__(self, model):\n", - " pass\n", + " return\n", " \n", " def fit(self, epochs, train_dataloader, val_dataloader, optimizer, criterion, device, early_stopping_limit = 0):\n", " return\n", @@ -1370,7 +1371,8 @@ "- ```A.ColorJitter``` for color jittering.\n", "\n", "Albumentations can also be used for image normalization (```A.Normalize```), resizing (```A.Resize```), and converting images to PyTorch tensors with the (Channel, Height, Width) format using ```A.ToTensorV2```, which is required for model training.\n", - "Apply the following transformations only to the training set, as the validation set should remain as close as possible to the test set. Therefore, no transformations should be applied to it.\n", + "\n", + "**NOTE: Apply the following transformations only to the training set, as the validation set should remain as close as possible to the test set. Therefore, no transformations should be applied to it.**\n", "\n", "```python\n", "A.Affine(scale = (0.2, 1.5), p = 0.1),\n", @@ -1493,7 +1495,7 @@ "id": "82", "metadata": {}, "source": [ - "### Model Training Overview\n", + "#### Model Training Overview\n", "\n", "Model training involves a sequence of key steps.\n", "The first step is to check which computational devices are available.\n", @@ -1576,6 +1578,8 @@ "source": [ "#### Loss function\n", "\n", + "In this notebook, we use cross entropy loss, which is the standard loss function for classification tasks. To build intuition, imagine taking a multiple-choice exam where you do not just pick a single answer, but instead assign a confidence score (probability) to every available option. Cross entropy acts as a very strict grader. If the correct answer is 'Dog', and your model is 99% confident it is a 'Dog', the grader gives a penalty (loss) close to zero. However, if the model is only 10% confident it is a 'Dog', the penalty increases because the model was unsure. Crucially, if the model is 99% confident it is a 'Cat' when it is actually a 'Dog', the penalty skyrockets. Cross entropy heavily penalizes a model for being confidently wrong.\n", + "\n", "The cross entropy loss function is defined by:\n", "\n", "$$\n", @@ -1610,7 +1614,21 @@ "id": "89", "metadata": {}, "source": [ - "#### Initialise model architecture" + "#### Initialise model architecture\n", + "\n", + "In this notebook, we are using a Convolutional Neural Network (CNN) as our model architecture. CNNs are deep learning models specifically designed to process visual data, like images. Their core engine is the convolution operation. Imagine sliding a small magnifying glass (called a filter) across an image, step-by-step. Instead of trying to look at the whole picture at once, these filters analyze small, localized patches to detect specific patterns.\n", + "\n", + "As the image passes through the network, a step-by-step recognition process happens: early layers detect simple lines and edges, middle layers combine those into textures or object parts (like a car tire or a dog's ear), and deep layers assemble them to recognize complex shapes.\n", + "\n", + "The overall pipeline looks like this:\n", + "\n", + "1. Start with the raw image input.\n", + "\n", + "2. Pass it through these convolutional blocks—which often shrink the data to keep only the most important details and save memory.\n", + "\n", + "3. Flatten the resulting 2D maps into a 1D vector.\n", + "\n", + "4. Feed that list into a standard classifier to make the final prediction." ] }, { @@ -1630,7 +1648,13 @@ "source": [ "#### Optimiser function\n", "\n", - "In this notebook, we are using Adam optimiser (```optimizer = optim.Adam(model.parameters(), lr = LR)```) which is one of the most used optimisers in deep neural network optimisation (see [Gentle Introduction to the Adam Optimisation Algorithm for Deep Learning](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n", + "In this notebook, we are using Adam (Adaptive Moment Estimation) optimiser (```optimizer = optim.Adam(model.parameters(), lr = LR)```) which is one of the most used optimisers in deep neural network optimisation (see [Gentle Introduction to the Adam Optimisation Algorithm for Deep Learning](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n", + "\n", + "To understand Adam, it helps to visualize a heavy ball rolling down a bumpy hill toward a valley (the minimum loss). In standard gradient descent, the ball relies only on the gradient ($g_t$) at its exact current location. This can be inefficient, causing it to zig-zag wildly across steep ravines or slow to a crawl on flat plateaus. Adam solves this by keeping track of two historical records, named \"moments\", to guide the ball more intelligently.\n", + "\n", + "First, Adam uses momentum ($m_t$). Just like a heavy ball builds up physical momentum and barrels through tiny bumps without getting thrown off course, Adam remembers the direction of past gradients to maintain a smooth, forward-moving trajectory. Second, it uses an adaptive step size based on the terrain ($v_t$). If the gradient in a specific direction is consistently huge, Adam scales down the learning rate ($\\alpha$) for that parameter so it takes smaller, careful steps and avoids overshooting the valley. Conversely, for parameters with very small, flat gradients, it increases the step size to speed up the journey. The bias corrections ($\\hat{m}_t$ and $\\hat{v}_t$) are simply included to ensure the ball doesn't start its descent too sluggishly from a dead stop.\n", + "\n", + "Mathematically, the parameter update at each step is given by:\n", "\n", "The parameter update at each step is given by:\n", "\n", @@ -2020,7 +2044,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/data/data_exploration/bubbly.py b/data/data_exploration/bubbly.py index ee6f6cde..1fd259f0 100644 --- a/data/data_exploration/bubbly.py +++ b/data/data_exploration/bubbly.py @@ -269,18 +269,20 @@ def make_grid(dataset, column_names, time_column, years=None): # Each column name is unique temp = col_name_template.format(year, col_name) if dataset_by_year[col_name].size != 0: - grid = grid.append( - {"value": list(dataset_by_year[col_name]), "key": temp}, - ignore_index=True, + new_grid = [] + new_grid.append( + {"value": list(dataset_by_year[col_name]), "key": temp} ) + new_grid_df = pd.DataFrame(new_grid) + grid = pd.concat([grid, new_grid_df], ignore_index=True) else: # Check if this can be simplified for col_name in column_names: # Each column name is unique - grid = grid.append( - {"value": list(dataset[col_name]), "key": col_name + "_grid"}, - ignore_index=True, - ) + new_grid = [] + new_grid.append({"value": list(dataset_by_year[col_name]), "key": temp}) + new_grid_df = pd.DataFrame(new_grid) + grid = pd.concat([grid, new_grid_df], ignore_index=True) return grid @@ -310,13 +312,15 @@ def make_grid_with_categories( # Each column name is unique temp = col_name_template.format(year, col_name, category) if dataset_by_year_and_cat[col_name].size != 0: - grid = grid.append( + new_grid = [] + new_grid.append( { "value": list(dataset_by_year_and_cat[col_name]), "key": temp, - }, - ignore_index=True, + } ) + new_grid_df = pd.DataFrame(new_grid) + grid = pd.concat([grid, new_grid_df], ignore_index=True) else: col_name_template = "{}+{}_grid" for category in categories: @@ -325,10 +329,12 @@ def make_grid_with_categories( # Each column name is unique temp = col_name_template.format(col_name, category) if dataset_by_cat[col_name].size != 0: - grid = grid.append( - {"value": list(dataset_by_cat[col_name]), "key": temp}, - ignore_index=True, + new_grid = [] + new_grid.append( + {"value": list(dataset_by_cat[col_name]), "key": temp} ) + new_grid_df = pd.DataFrame(new_grid) + grid = pd.concat([grid, new_grid_df], ignore_index=True) return grid From e5b12caf69b4d0cc06e5fb74e21be63bcef1efe3 Mon Sep 17 00:00:00 2001 From: Lopes Date: Fri, 10 Apr 2026 11:52:49 +0200 Subject: [PATCH 2/6] Add hint functions to the image processing part --- 31_image_classification.ipynb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/31_image_classification.ipynb b/31_image_classification.ipynb index d9cce389..2f645e57 100644 --- a/31_image_classification.ipynb +++ b/31_image_classification.ipynb @@ -709,10 +709,10 @@ "They help the model become invariant to different orientations and scales:\n", "\n", "- **Scaling**: Resizes the image to a specific size, often required to match input dimensions for image classifiers.\n", - " It uses interpolation to obtain the new pixel-values.\n", - "- **Cropping**: Extracts a subregion of the image; useful for focusing on important parts or adding variability.\n", - "- **Horizontal and vertical flip**: Flips the image along the x-axis or y-axis; helps the model learn symmetry.\n", - "- **Rotation**: Rotates the image by a small angle to simulate different orientations of the objects." + " It uses interpolation to obtain the new pixel-values. Use ```cv2.resize()```\n", + "- **Cropping**: Extracts a subregion of the image; useful for focusing on important parts or adding variability. No openCV function needed.\n", + "- **Horizontal and vertical flip**: Flips the image along the x-axis or y-axis; helps the model learn symmetry. Use ```cv2.flip()```\n", + "- **Rotation**: Rotates the image by a small angle to simulate different orientations of the objects. Use ```cv2.getRotationMatrix2D``` and ```cv2.warpAffine()```" ] }, { @@ -970,9 +970,9 @@ "Filtering helps reduce noise and enhance specific image features.\n", "These are often used as a form of preprocessing before feeding images into a model:\n", "\n", - "- **Average filter**: Applies a smoothing effect by replacing each pixel with the average of its neighborhood.\n", - "- **Median filter**: Reduces salt-and-pepper noise by replacing each pixel with the median of neighboring pixels.\n", - "- **Gaussian filter**: Applies a Gaussian blur to smooth the image, often used to reduce high-frequency noise." + "- **Average filter**: Applies a smoothing effect by replacing each pixel with the average of its neighborhood. Use ```cv2.blur()```.\n", + "- **Median filter**: Reduces salt-and-pepper noise by replacing each pixel with the median of neighboring pixels. Use ```cv2.medianBlur()```.\n", + "- **Gaussian filter**: Applies a Gaussian blur to smooth the image, often used to reduce high-frequency noise. Use ```cv2.GaussianBlur()```." ] }, { @@ -1131,9 +1131,9 @@ "\n", "Photometric transformations modify the color properties of an image to simulate different lighting conditions and improve model robustness to brightness and contrast changes:\n", "\n", - "- **Brightness**: Randomly increases or decreases the brightness of the image.\n", - "- **Contrast**: Alters the difference between light and dark regions in the image.\n", - "- **Saturation**: Modifies the intensity of the colors in the image." + "- **Brightness**: Randomly increases or decreases the brightness of the image. Use ```cv2.convertScaleAbs()```.\n", + "- **Contrast**: Alters the difference between light and dark regions in the image. Use ```cv2.convertScaleAbs()```.\n", + "- **Saturation**: Modifies the intensity of the colors in the image. Use ```cv2.cvtColor(), cv2.split(), and cv2.merge()```." ] }, { From e0d3c00c114ec2ad659e921f6e0f5c7fb8bf452c Mon Sep 17 00:00:00 2001 From: Lopes Date: Fri, 10 Apr 2026 11:58:12 +0200 Subject: [PATCH 3/6] Add missing parenthesis to a hint function --- 31_image_classification.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/31_image_classification.ipynb b/31_image_classification.ipynb index 2f645e57..14991083 100644 --- a/31_image_classification.ipynb +++ b/31_image_classification.ipynb @@ -712,7 +712,7 @@ " It uses interpolation to obtain the new pixel-values. Use ```cv2.resize()```\n", "- **Cropping**: Extracts a subregion of the image; useful for focusing on important parts or adding variability. No openCV function needed.\n", "- **Horizontal and vertical flip**: Flips the image along the x-axis or y-axis; helps the model learn symmetry. Use ```cv2.flip()```\n", - "- **Rotation**: Rotates the image by a small angle to simulate different orientations of the objects. Use ```cv2.getRotationMatrix2D``` and ```cv2.warpAffine()```" + "- **Rotation**: Rotates the image by a small angle to simulate different orientations of the objects. Use ```cv2.getRotationMatrix2D()``` and ```cv2.warpAffine()```" ] }, { From efe391d828895083e71ccabb2fa1e880ba01c452 Mon Sep 17 00:00:00 2001 From: Lopes Date: Fri, 10 Apr 2026 14:15:25 +0200 Subject: [PATCH 4/6] Automatic access to the dataset --- 31_image_classification.ipynb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/31_image_classification.ipynb b/31_image_classification.ipynb index 14991083..82a1b8d4 100644 --- a/31_image_classification.ipynb +++ b/31_image_classification.ipynb @@ -553,7 +553,7 @@ "\n", "class Trainer():\n", " def __init__(self, model):\n", - " return\n", + " pass\n", " \n", " def fit(self, epochs, train_dataloader, val_dataloader, optimizer, criterion, device, early_stopping_limit = 0):\n", " return\n", @@ -619,8 +619,7 @@ "source": [ "### Load data\n", "\n", - "Training and test sets are loaded using Pickle library. If you do not have the dataset already, open this [link](https://www.dropbox.com/scl/fo/p7gfb0kpgkbrrjup340pi/AAkX2u1g-W7290-Aq7gHHvo?rlkey=vdxaj6npfy09ywh17nl8f9v6e&st=8hfq9z20&dl=0) and download it.\n", - "Place it inside the data folder." + "Training and test sets are loaded using Pickle library." ] }, { @@ -630,12 +629,18 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", + "from pathlib import Path\n", + "\n", + "# Get current directory\n", + "current_dir = Path.cwd()\n", + "\n", + "# Go up one level to /home/jovyan, then into datasets\n", + "datasets_dir = current_dir.parent / \"datasets\"\n", "\n", "# Sets filepaths\n", - "dataset_folder = os.path.join(\"data/CIFAR10\")\n", - "train_set_file = os.path.join(dataset_folder, \"train_set.pkl\")\n", - "test_set_file = os.path.join(dataset_folder, \"test_set.pkl\")\n", + "dataset_folder = datasets_dir / \"CIFAR10\"\n", + "train_set_file = dataset_folder / \"train_set.pkl\"\n", + "test_set_file = dataset_folder / \"test_set.pkl\"\n", "\n", "# Load sets\n", "train_set = load_pickle_file(train_set_file)\n", @@ -1749,7 +1754,7 @@ "import os\n", "import torch\n", "\n", - "# Model filename\n", + "# Model filename (In case you want to use the already trained model, replace this by model_path = dataset_folder / \"cnn_weights.pt\")\n", "model_path = \"cnn_weights.pt\"\n", "\n", "if os.path.exists(model_path):\n", @@ -1785,7 +1790,7 @@ "import pandas as pd\n", "from matplotlib import pyplot as plt\n", "\n", - "# Load the training log file\n", + "# Load the training log file (In case you want to use the already trained model, replace this by model_path = dataset_folder / \"training_log.txt\")\n", "training_log = None\n", "\n", "plt.figure()\n", From 4d2b1b7d82bbf83fdf9430093dc19b1b07d0da99 Mon Sep 17 00:00:00 2001 From: Lopes Date: Mon, 13 Apr 2026 11:58:37 +0200 Subject: [PATCH 5/6] Restore introduction to data exploration files --- 30_introduction_data_exploration.ipynb | 60 ++++++-------------------- data/data_exploration/bubbly.py | 32 ++++++-------- 2 files changed, 26 insertions(+), 66 deletions(-) diff --git a/30_introduction_data_exploration.ipynb b/30_introduction_data_exploration.ipynb index 46626058..01ec0236 100644 --- a/30_introduction_data_exploration.ipynb +++ b/30_introduction_data_exploration.ipynb @@ -334,7 +334,9 @@ " Returns:\n", " pd.DataFrame: A DataFrame containing the happiness data.\n", " \"\"\"\n", - " return" + " # Your code starts here\n", + " return\n", + " # Your code ends here" ] }, { @@ -591,33 +593,9 @@ " Returns:\n", " - Cleaned DataFrame with missing values filled\n", " \"\"\"\n", - " min_year = happiness_df[\"year\"].min()\n", - " max_year = happiness_df[\"year\"].max()\n", - " possible_years = np.arange(min_year, max_year + 1)\n", - " possible_countries = happiness_df[\"Country name\"].unique()\n", - "\n", - " # Create entries for each country for every year\n", - " all_years_countries = pd.DataFrame(\n", - " [(country, year) for country in possible_countries for year in possible_years],\n", - " columns=[\"Country name\", \"year\"],\n", - " )\n", - " # Extend the happiness_df to include all years for each country\n", - "\n", - " complete_happiness_df = all_years_countries.merge(\n", - " happiness_df, on=[\"Country name\", \"year\"], how=\"left\"\n", - " )\n", - "\n", - " # Set initial values to 1:\n", - " year_2005 = complete_happiness_df[\"year\"] == 2005\n", - " complete_happiness_df.loc[year_2005] = complete_happiness_df.loc[year_2005].fillna(\n", - " 1\n", - " )\n", - " # Apply forward fill for any remaining NaNs (make sure sorting is in the right order)\n", - " complete_happiness_df = complete_happiness_df.sort_values(\n", - " by=[\"Country name\", \"year\"]\n", - " ).ffill()\n", - "\n", - " return complete_happiness_df" + " # Your code starts here\n", + " return \n", + " # Your code ends here" ] }, { @@ -729,9 +707,9 @@ " Returns:\n", " - DataFrame with the regional indicator added\n", " \"\"\"\n", - " final_happiness_df = pd.merge(cleaned_happiness_df, region_df, on='Country name', how='left')\n", - " final_happiness_df['Regional indicator'] = final_happiness_df['Regional indicator'].fillna('Unknown')\n", - " return final_happiness_df" + " # Your code starts here\n", + " return\n", + " # Your code ends here" ] }, { @@ -1235,14 +1213,9 @@ " Returns:\n", " - Dictionary containing the trace and frame information\n", " \"\"\"\n", - " \"\"\"Make a frame for a given year with bubble size\"\"\"\n", - "\n", - " def trace_by_category(dataset, year, x_column, y_column, description_column, category_column, category, bubble_size_column):\n", - " \"\"\"Make a trace for a given year with bubble size\"\"\"\n", - " trace = {'x': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), x_column]), 'y': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), y_column]), 'mode': 'markers', 'text': list(dataset.loc[dataset['year'] == year, description_column]), 'marker': {'size': list(dataset.loc[(dataset['year'] == year) & (dataset[category_column] == category), bubble_size_column]), 'sizemode': 'area', 'sizeref': 1}, 'type': 'scatter', 'name': category}\n", - " return trace\n", - " frame = {'data': [trace_by_category(dataset, year, x_column, y_column, description_column, category_column, category, bubble_size_column) for category in dataset[category_column].unique()], 'name': str(year)}\n", - " return frame" + " # Your code starts here\n", + " return\n", + " # Your code ends here" ] }, { @@ -1317,13 +1290,6 @@ " x_logscale=True, scale_bubble=3, height=650)\n", "iplot(figure, config={'scrollZoom': True})\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -1342,7 +1308,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.12.10" } }, "nbformat": 4, diff --git a/data/data_exploration/bubbly.py b/data/data_exploration/bubbly.py index 1fd259f0..ee6f6cde 100644 --- a/data/data_exploration/bubbly.py +++ b/data/data_exploration/bubbly.py @@ -269,20 +269,18 @@ def make_grid(dataset, column_names, time_column, years=None): # Each column name is unique temp = col_name_template.format(year, col_name) if dataset_by_year[col_name].size != 0: - new_grid = [] - new_grid.append( - {"value": list(dataset_by_year[col_name]), "key": temp} + grid = grid.append( + {"value": list(dataset_by_year[col_name]), "key": temp}, + ignore_index=True, ) - new_grid_df = pd.DataFrame(new_grid) - grid = pd.concat([grid, new_grid_df], ignore_index=True) else: # Check if this can be simplified for col_name in column_names: # Each column name is unique - new_grid = [] - new_grid.append({"value": list(dataset_by_year[col_name]), "key": temp}) - new_grid_df = pd.DataFrame(new_grid) - grid = pd.concat([grid, new_grid_df], ignore_index=True) + grid = grid.append( + {"value": list(dataset[col_name]), "key": col_name + "_grid"}, + ignore_index=True, + ) return grid @@ -312,15 +310,13 @@ def make_grid_with_categories( # Each column name is unique temp = col_name_template.format(year, col_name, category) if dataset_by_year_and_cat[col_name].size != 0: - new_grid = [] - new_grid.append( + grid = grid.append( { "value": list(dataset_by_year_and_cat[col_name]), "key": temp, - } + }, + ignore_index=True, ) - new_grid_df = pd.DataFrame(new_grid) - grid = pd.concat([grid, new_grid_df], ignore_index=True) else: col_name_template = "{}+{}_grid" for category in categories: @@ -329,12 +325,10 @@ def make_grid_with_categories( # Each column name is unique temp = col_name_template.format(col_name, category) if dataset_by_cat[col_name].size != 0: - new_grid = [] - new_grid.append( - {"value": list(dataset_by_cat[col_name]), "key": temp} + grid = grid.append( + {"value": list(dataset_by_cat[col_name]), "key": temp}, + ignore_index=True, ) - new_grid_df = pd.DataFrame(new_grid) - grid = pd.concat([grid, new_grid_df], ignore_index=True) return grid From 276d088f31e0d5d1d730572361c356d6cafde8da Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 14 Apr 2026 16:30:54 +0200 Subject: [PATCH 6/6] Small changes --- 31_image_classification.ipynb | 127 ++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 52 deletions(-) diff --git a/31_image_classification.ipynb b/31_image_classification.ipynb index 82a1b8d4..da396ac8 100644 --- a/31_image_classification.ipynb +++ b/31_image_classification.ipynb @@ -160,7 +160,7 @@ "image = self.images[idx]\n", "label = self.labels[idx]\n", "\n", - "# Pytorch expects images with the shape (Channels, Height, Width). However, Albumentations library (library used for image augmentation) requires them with the shape (Height, Width, Channels). Therefore, we need to use transpose to make them compatible with the library. Later in the notebook the images are converted back to the original shape by using A.ToTensorV2().\n", + "# PyTorch expects images with the shape (Channels, Height, Width). However, Albumentations library (library used for image augmentation) requires them with the shape (Height, Width, Channels). Therefore, we need to use transpose to make them compatible with the library. Later in the notebook the images are converted back to the original shape by using A.ToTensorV2().\n", "image = np.transpose(image, (1, 2, 0))\n", "\n", "# Apply transformations on the images\n", @@ -232,7 +232,7 @@ "```\n", "\n", "This layer normalizes the outputs, making training faster and more stable.\n", - "The combination of the foreamentioned layers is also usually called as convolutional block.\n", + "The combination of the aforementioned layers is also usually called as convolutional block.\n", "After defining the first convolutional block, lets define the second one:\n", "\n", "```python\n", @@ -271,7 +271,7 @@ "```python\n", "self.dropout = nn.Dropout(p = 0.3)\n", "```\n", - "This layer randomly turns off a pre-define percentage of neurons (`p = 0.3`) during training to prevent overfitting — so the model does not memorize the training data too closely.\n", + "This layer randomly turns off a pre-defined percentage of neurons (`p = 0.3`) during training to prevent overfitting — so the model does not memorize the training data too closely.\n", "Finally, we define the classifier:\n", "\n", "```python\n", @@ -559,7 +559,8 @@ " return\n", " \n", " def predict(self, test_dataloader, device):\n", - " return\n" + " return\n", + "\n" ] }, { @@ -665,7 +666,7 @@ "These methods can also simulate real-world variability, helping models generalize better. \n", "\n", "In this notebook, we explore three categories of image transformations: **geometric transformations**, **image filtering**, and **photometric transformations**.\n", - "The following cells contain a series of exercicies designed to help you explore the OpenCV-Python library. \n", + "The following cells contain a series of exercises designed to help you explore the OpenCV-Python library.\n", "\n", "If you are unfamiliar with a particular method, refer to the [Image Processing in OpenCV](https://docs.opencv.org/4.x/d2/d96/tutorial_py_table_of_contents_imgproc.html) documentation.\n", "There you can find the description of the functions needed for [Geometric transformations](https://docs.opencv.org/4.x/da/d6e/tutorial_py_geometric_transformations.html) and [image filtering](https://docs.opencv.org/4.x/d4/d13/tutorial_py_filtering.html).\n", @@ -1377,8 +1378,17 @@ "\n", "Albumentations can also be used for image normalization (```A.Normalize```), resizing (```A.Resize```), and converting images to PyTorch tensors with the (Channel, Height, Width) format using ```A.ToTensorV2```, which is required for model training.\n", "\n", - "**NOTE: Apply the following transformations only to the training set, as the validation set should remain as close as possible to the test set. Therefore, no transformations should be applied to it.**\n", - "\n", + "
\n", + " The train_transform defined in the next cell only contains the preprocessing steps needed to feed images into the model (A.Normalize, A.Resize, A.ToTensorV2). Extend it by adding one or more augmentation operations from the list above — for example, the snippet below is a reasonable starting point. Insert your augmentations before A.Normalize so that they are applied to the raw image.

\n", + "Important: Apply augmentations only to the training set. The validation and test transforms (val_transform) should stay as preprocessing only, so that evaluation is performed on images that match the real distribution as closely as possible.\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "76", + "metadata": {}, + "source": [ "```python\n", "A.Affine(scale = (0.2, 1.5), p = 0.1),\n", "A.Rotate(limit = 45, p = 0.1),\n", @@ -1391,7 +1401,7 @@ { "cell_type": "code", "execution_count": null, - "id": "76", + "id": "77", "metadata": {}, "outputs": [], "source": [ @@ -1415,7 +1425,7 @@ }, { "cell_type": "markdown", - "id": "77", + "id": "78", "metadata": {}, "source": [ "#### PyTorch Datasets\n", @@ -1426,7 +1436,7 @@ { "cell_type": "code", "execution_count": null, - "id": "78", + "id": "79", "metadata": {}, "outputs": [], "source": [ @@ -1438,7 +1448,7 @@ }, { "cell_type": "markdown", - "id": "79", + "id": "80", "metadata": {}, "source": [ "#### PyTorch Dataloaders\n", @@ -1455,7 +1465,7 @@ { "cell_type": "code", "execution_count": null, - "id": "80", + "id": "81", "metadata": {}, "outputs": [], "source": [ @@ -1470,7 +1480,7 @@ }, { "cell_type": "markdown", - "id": "81", + "id": "82", "metadata": {}, "source": [ "### Model training\n", @@ -1480,7 +1490,7 @@ "1. First, we must check which devices are available for training the model.\n", " In case a GPU with Cuda cores is available is should be used as it really improves the speed.\n", " Otherwise, lets use CPU. \n", - "1. Then, model and training hyperparameters should be defined, such as numer of output classes, number of training epochs, number of consecutive not improving epochs needed for stopping the training in case we use early stopping regularisation, and learning rate.\n", + "1. Then, model and training hyperparameters should be defined, such as number of output classes, number of training epochs, number of consecutive not improving epochs needed for stopping the training in case we use early stopping regularisation, and learning rate.\n", " Other hyperparameters can be defined, it depends on what the user wants to do during the training.\n", " In this notebook we are going to define the number of epochs, which are the number of times the model is going to see the training set.\n", " Early stopping is a way of trying to avoid overfitting where the model evaluates the model every new epoch using a validation set.\n", @@ -1497,7 +1507,7 @@ }, { "cell_type": "markdown", - "id": "82", + "id": "83", "metadata": {}, "source": [ "#### Model Training Overview\n", @@ -1533,7 +1543,7 @@ }, { "cell_type": "markdown", - "id": "83", + "id": "84", "metadata": {}, "source": [ "#### Check which device is used for training" @@ -1542,18 +1552,20 @@ { "cell_type": "code", "execution_count": null, - "id": "84", - "metadata": {}, + "id": "85", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "import torch\n", "\n", - "# Check which device is available for training the model\n" + "# Check which device is available for training the model" ] }, { "cell_type": "markdown", - "id": "85", + "id": "86", "metadata": {}, "source": [ "#### Define training hyperparameters" @@ -1562,7 +1574,7 @@ { "cell_type": "code", "execution_count": null, - "id": "86", + "id": "87", "metadata": {}, "outputs": [], "source": [ @@ -1578,12 +1590,19 @@ }, { "cell_type": "markdown", - "id": "87", + "id": "88", "metadata": {}, "source": [ "#### Loss function\n", "\n", - "In this notebook, we use cross entropy loss, which is the standard loss function for classification tasks. To build intuition, imagine taking a multiple-choice exam where you do not just pick a single answer, but instead assign a confidence score (probability) to every available option. Cross entropy acts as a very strict grader. If the correct answer is 'Dog', and your model is 99% confident it is a 'Dog', the grader gives a penalty (loss) close to zero. However, if the model is only 10% confident it is a 'Dog', the penalty increases because the model was unsure. Crucially, if the model is 99% confident it is a 'Cat' when it is actually a 'Dog', the penalty skyrockets. Cross entropy heavily penalizes a model for being confidently wrong.\n", + "In this notebook, we use cross entropy loss, which is the standard loss function for classification tasks.\n", + "\n", + "To build intuition, imagine taking a multiple-choice exam where you do not just pick a single answer, but instead assign a confidence score (probability) to every available option.\n", + "Cross entropy acts as a grader that only looks at the probability you assigned to the correct answer: the higher that number, the smaller your penalty.\n", + "If the correct answer is 'Dog', and your model is 99% confident it is a 'Dog', the penalty is close to zero.\n", + "If the model is only 10% confident it is a 'Dog', the penalty is much larger: the grader does not care that the remaining 90% was spread over wrong answers, only that 'Dog' got 10%.\n", + "Crucially, if the model is 99% confident it is a 'Cat' when it is actually a 'Dog', only 1% was left for 'Dog', so the penalty skyrockets.\n", + "Cross entropy heavily penalizes a model for being confidently wrong.\n", "\n", "The cross entropy loss function is defined by:\n", "\n", @@ -1605,18 +1624,20 @@ { "cell_type": "code", "execution_count": null, - "id": "88", - "metadata": {}, + "id": "89", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "import torch.nn as nn\n", "\n", - "# Initialise the Cross Entropy Loss and send it to the training device\n" + "# Initialise the Cross Entropy Loss and send it to the training device" ] }, { "cell_type": "markdown", - "id": "89", + "id": "90", "metadata": {}, "source": [ "#### Initialise model architecture\n", @@ -1639,7 +1660,7 @@ { "cell_type": "code", "execution_count": null, - "id": "90", + "id": "91", "metadata": {}, "outputs": [], "source": [ @@ -1648,7 +1669,7 @@ }, { "cell_type": "markdown", - "id": "91", + "id": "92", "metadata": {}, "source": [ "#### Optimiser function\n", @@ -1697,18 +1718,20 @@ { "cell_type": "code", "execution_count": null, - "id": "92", - "metadata": {}, + "id": "93", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "import torch.optim as optim\n", "\n", - "# Initialise the Adam optimiser\n" + "# Initialise the Adam optimiser" ] }, { "cell_type": "markdown", - "id": "93", + "id": "94", "metadata": {}, "source": [ "#### Train model\n", @@ -1721,7 +1744,7 @@ { "cell_type": "code", "execution_count": null, - "id": "94", + "id": "95", "metadata": {}, "outputs": [], "source": [ @@ -1730,7 +1753,7 @@ }, { "cell_type": "markdown", - "id": "95", + "id": "96", "metadata": {}, "source": [ "After initialising the trainer instance, check whether a trained model already exists.\n", @@ -1747,7 +1770,7 @@ { "cell_type": "code", "execution_count": null, - "id": "96", + "id": "97", "metadata": {}, "outputs": [], "source": [ @@ -1765,7 +1788,7 @@ }, { "cell_type": "markdown", - "id": "97", + "id": "98", "metadata": {}, "source": [ "#### Learning curves\n", @@ -1783,7 +1806,7 @@ { "cell_type": "code", "execution_count": null, - "id": "98", + "id": "99", "metadata": {}, "outputs": [], "source": [ @@ -1804,7 +1827,7 @@ }, { "cell_type": "markdown", - "id": "99", + "id": "100", "metadata": {}, "source": [ "### Model testing\n", @@ -1825,7 +1848,7 @@ { "cell_type": "code", "execution_count": null, - "id": "100", + "id": "101", "metadata": {}, "outputs": [], "source": [ @@ -1834,7 +1857,7 @@ }, { "cell_type": "markdown", - "id": "101", + "id": "102", "metadata": {}, "source": [ "### Explore results\n", @@ -1846,7 +1869,7 @@ }, { "cell_type": "markdown", - "id": "102", + "id": "103", "metadata": {}, "source": [ "#### Compute average accuracy\n", @@ -1857,7 +1880,7 @@ { "cell_type": "code", "execution_count": null, - "id": "103", + "id": "104", "metadata": {}, "outputs": [], "source": [ @@ -1868,7 +1891,7 @@ }, { "cell_type": "markdown", - "id": "104", + "id": "105", "metadata": {}, "source": [ "#### Compute confusion matrix\n", @@ -1883,7 +1906,7 @@ { "cell_type": "code", "execution_count": null, - "id": "105", + "id": "106", "metadata": {}, "outputs": [], "source": [ @@ -1918,7 +1941,7 @@ }, { "cell_type": "markdown", - "id": "106", + "id": "107", "metadata": {}, "source": [ "### Explain image classifier predictions\n", @@ -1931,7 +1954,7 @@ }, { "cell_type": "markdown", - "id": "107", + "id": "108", "metadata": {}, "source": [ "#### Prepare image for Grad-CAM\n", @@ -1947,7 +1970,7 @@ { "cell_type": "code", "execution_count": null, - "id": "108", + "id": "109", "metadata": {}, "outputs": [], "source": [ @@ -1963,7 +1986,7 @@ }, { "cell_type": "markdown", - "id": "109", + "id": "110", "metadata": {}, "source": [ "#### Compute GradCAM heatmap\n", @@ -1987,7 +2010,7 @@ { "cell_type": "code", "execution_count": null, - "id": "110", + "id": "111", "metadata": {}, "outputs": [], "source": [ @@ -2005,7 +2028,7 @@ }, { "cell_type": "markdown", - "id": "111", + "id": "112", "metadata": {}, "source": [ "#### Visualise Grad-CAM heatmap with the image\n", @@ -2017,7 +2040,7 @@ { "cell_type": "code", "execution_count": null, - "id": "112", + "id": "113", "metadata": {}, "outputs": [], "source": [