{ "cells": [ { "cell_type": "code", "source": [ "import pandas as pd\n", "\n", "pd.set_option('display.max_rows', None)\n", "pd.set_option('display.max_columns', None)\n", "\n", "# Read the CSV file into a DataFrame\n", "df = pd.read_csv('Data (1).csv')\n", "\n", "# Display the first 5 rows\n", "print(df.head(10).to_markdown(index=False, numalign=\"left\", stralign=\"left\"))\n", "\n", "# Print the column names and their data types\n", "print(df.info())" ], "metadata": { "id": "SNY3X6KTc3wc" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "df['Age'].mean()\n", "df['Age'].median()" ], "metadata": { "id": "x0gB8Gg1SD2N" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Fill in missing values in `Age` and `Salary` with their respective means\n", "df['Age'].fillna(df['Age'].median(), inplace=True)\n", "df['Salary'].fillna(df['Salary'].median(), inplace=True)\n", "\n", "# Print the first 5 rows of the dataframe\n", "print(df.head(10).to_markdown(index=False, numalign=\"left\", stralign=\"left\"))" ], "outputs": [], "execution_count": null, "metadata": { "id": "whwSJ5vmc2I_" } }, { "cell_type": "markdown", "source": [ "# Encoding" ], "metadata": { "id": "bYiZ0zmHZmr5" } }, { "cell_type": "code", "source": [ "import pandas as pd\n", "from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder\n", "\n", "# Creating the updated dummy data\n", "data = {\n", " 'ID': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n", " 'Age Group': ['Young', 'Young', 'Middle-Aged', 'Senior', 'Middle-Aged',\n", " 'Young', 'Senior', 'Middle-Aged', 'Young', 'Senior'],\n", " 'Education Level': ['High School', 'Bachelor', 'High School', 'Master', 'PhD',\n", " 'High School', 'Bachelor', 'Master', 'PhD', 'PhD'],\n", " 'City': ['New York', 'Los Angeles', 'Chicago', 'New York', 'Los Angeles',\n", " 'Chicago', 'Miami', 'New York', 'Miami', 'Chicago']\n", "}\n", "\n", "# Creating DataFrame\n", "df = pd.DataFrame(data)\n", "\n", "df" ], "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " ID Age Group Education Level City\n", "0 1 Young High School New York\n", "1 2 Young Bachelor Los Angeles\n", "2 3 Middle-Aged High School Chicago\n", "3 4 Senior Master New York\n", "4 5 Middle-Aged PhD Los Angeles\n", "5 6 Young High School Chicago\n", "6 7 Senior Bachelor Miami\n", "7 8 Middle-Aged Master New York\n", "8 9 Young PhD Miami\n", "9 10 Senior PhD Chicago" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDAge GroupEducation LevelCity
01YoungHigh SchoolNew York
12YoungBachelorLos Angeles
23Middle-AgedHigh SchoolChicago
34SeniorMasterNew York
45Middle-AgedPhDLos Angeles
56YoungHigh SchoolChicago
67SeniorBachelorMiami
78Middle-AgedMasterNew York
89YoungPhDMiami
910SeniorPhDChicago
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", " \n", " \n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"ID\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3,\n \"min\": 1,\n \"max\": 10,\n \"num_unique_values\": 10,\n \"samples\": [\n 9,\n 2,\n 6\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Age Group\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Young\",\n \"Middle-Aged\",\n \"Senior\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Education Level\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"Bachelor\",\n \"PhD\",\n \"High School\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"City\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"Los Angeles\",\n \"Miami\",\n \"New York\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 8 } ], "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 363 }, "id": "b-ShFJ_ic2JB", "outputId": "0ffbe8d6-26be-4aac-dbd1-7cf1b1e264ba" } }, { "cell_type": "code", "source": [ "# Initializing OrdinalEncoder for Age Group and Education Level\n", "ordinal_encoder = OrdinalEncoder(categories=[\n", " ['Young', 'Middle-Aged', 'Senior'], # Age Group order\n", " ['High School', 'Bachelor', 'Master', 'PhD'], # Education Level order\n", "])\n", "\n", "# Encoding the ordinal columns\n", "df[['Age Group', 'Education Level']] = ordinal_encoder.fit_transform(\n", " df[['Age Group', 'Education Level']]\n", ")\n", "\n", "df" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 363 }, "id": "HUYq36FPVXeI", "outputId": "bd06fd62-2db9-4fac-ba46-c56b7120bce6" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " ID Age Group Education Level City\n", "0 1 0.0 0.0 New York\n", "1 2 0.0 1.0 Los Angeles\n", "2 3 1.0 0.0 Chicago\n", "3 4 2.0 2.0 New York\n", "4 5 1.0 3.0 Los Angeles\n", "5 6 0.0 0.0 Chicago\n", "6 7 2.0 1.0 Miami\n", "7 8 1.0 2.0 New York\n", "8 9 0.0 3.0 Miami\n", "9 10 2.0 3.0 Chicago" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDAge GroupEducation LevelCity
010.00.0New York
120.01.0Los Angeles
231.00.0Chicago
342.02.0New York
451.03.0Los Angeles
560.00.0Chicago
672.01.0Miami
781.02.0New York
890.03.0Miami
9102.03.0Chicago
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", " \n", " \n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"ID\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3,\n \"min\": 1,\n \"max\": 10,\n \"num_unique_values\": 10,\n \"samples\": [\n 9,\n 2,\n 6\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Age Group\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.8755950357709131,\n \"min\": 0.0,\n \"max\": 2.0,\n \"num_unique_values\": 3,\n \"samples\": [\n 0.0,\n 1.0,\n 2.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Education Level\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1.2692955176439846,\n \"min\": 0.0,\n \"max\": 3.0,\n \"num_unique_values\": 4,\n \"samples\": [\n 1.0,\n 3.0,\n 0.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"City\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"Los Angeles\",\n \"Miami\",\n \"New York\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 9 } ] }, { "cell_type": "code", "source": [ "# Using OneHotEncoder for the 'City' column (non-ordinal)\n", "onehot_encoder = OneHotEncoder()\n", "\n", "# One-Hot Encoding the 'City' column\n", "city_encoded = onehot_encoder.fit_transform(df[['City']])\n", "\n", "# Converting sparse matrix to a dense DataFrame\n", "city_encoded_df = pd.DataFrame(city_encoded.toarray(), columns=onehot_encoder.categories_[0])\n", "\n", "# Concatenating the one-hot encoded columns with the original DataFrame (excluding the original 'City' column)\n", "df = pd.concat([df, city_encoded_df], axis=1).drop('City', axis=1)\n", "\n", "# Display the updated DataFrame\n", "df" ], "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " ID Age Group Education Level Chicago Los Angeles Miami New York\n", "0 1 0.0 0.0 0.0 0.0 0.0 1.0\n", "1 2 0.0 1.0 0.0 1.0 0.0 0.0\n", "2 3 1.0 0.0 1.0 0.0 0.0 0.0\n", "3 4 2.0 2.0 0.0 0.0 0.0 1.0\n", "4 5 1.0 3.0 0.0 1.0 0.0 0.0\n", "5 6 0.0 0.0 1.0 0.0 0.0 0.0\n", "6 7 2.0 1.0 0.0 0.0 1.0 0.0\n", "7 8 1.0 2.0 0.0 0.0 0.0 1.0\n", "8 9 0.0 3.0 0.0 0.0 1.0 0.0\n", "9 10 2.0 3.0 1.0 0.0 0.0 0.0" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDAge GroupEducation LevelChicagoLos AngelesMiamiNew York
010.00.00.00.00.01.0
120.01.00.01.00.00.0
231.00.01.00.00.00.0
342.02.00.00.00.01.0
451.03.00.01.00.00.0
560.00.01.00.00.00.0
672.01.00.00.01.00.0
781.02.00.00.00.01.0
890.03.00.00.01.00.0
9102.03.01.00.00.00.0
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", " \n", " \n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"ID\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3,\n \"min\": 1,\n \"max\": 10,\n \"num_unique_values\": 10,\n \"samples\": [\n 9,\n 2,\n 6\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Age Group\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.8755950357709131,\n \"min\": 0.0,\n \"max\": 2.0,\n \"num_unique_values\": 3,\n \"samples\": [\n 0.0,\n 1.0,\n 2.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Education Level\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1.2692955176439846,\n \"min\": 0.0,\n \"max\": 3.0,\n \"num_unique_values\": 4,\n \"samples\": [\n 1.0,\n 3.0,\n 0.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Chicago\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.48304589153964794,\n \"min\": 0.0,\n \"max\": 1.0,\n \"num_unique_values\": 2,\n \"samples\": [\n 1.0,\n 0.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Los Angeles\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.42163702135578396,\n \"min\": 0.0,\n \"max\": 1.0,\n \"num_unique_values\": 2,\n \"samples\": [\n 1.0,\n 0.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Miami\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.42163702135578396,\n \"min\": 0.0,\n \"max\": 1.0,\n \"num_unique_values\": 2,\n \"samples\": [\n 1.0,\n 0.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"New York\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.48304589153964794,\n \"min\": 0.0,\n \"max\": 1.0,\n \"num_unique_values\": 2,\n \"samples\": [\n 0.0,\n 1.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 10 } ], "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 363 }, "id": "-VfyHHf7c2JB", "outputId": "04b28491-a196-434f-e419-11a447363fc8" } }, { "cell_type": "code", "source": [], "metadata": { "id": "XwcqRZbWYFX7" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "UYhfE5AsYFUL" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "xpuf99KlYFQ5" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "jv6ZKjKCYFNo" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "DHybdYeQYFKO" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "# Normalization" ], "metadata": { "id": "ixwzeqOydM-j" } }, { "cell_type": "code", "source": [ "from sklearn.preprocessing import MinMaxScaler\n", "\n", "# Define the numerical features\n", "numerical_features = ['Age', 'Salary']\n", "\n", "# Apply MinMaxScaler to the numerical features\n", "min_max_scaler = MinMaxScaler()\n", "df_encoded_min_max = df.copy()\n", "df_encoded_min_max[numerical_features] = min_max_scaler.fit_transform(df_encoded_min_max[numerical_features])\n", "\n", "# Print the first 5 rows of the dataframe\n", "print(df_encoded_min_max.head(10).to_markdown(index=False, numalign=\"left\", stralign=\"left\"))\n", "\n", "# Print the column names and their data types of the transformed dataframe\n", "print('\\n',df_encoded_min_max.info())" ], "outputs": [], "execution_count": null, "metadata": { "id": "K_H_sN2xc2JC" } }, { "cell_type": "markdown", "source": [ "# StandardScaler = (x - mean) / standard deviation" ], "metadata": { "id": "o61AW8S1XXnb" } }, { "cell_type": "code", "source": [ "from sklearn.preprocessing import StandardScaler\n", "\n", "# Apply StandardScaler to the numerical features\n", "standard_scaler = StandardScaler()\n", "df_encoded_standard = df.copy()\n", "df_encoded_standard[numerical_features] = standard_scaler.fit_transform(df_encoded_standard[numerical_features])\n", "\n", "# Print the first 5 rows of the dataframe\n", "print(df_encoded_standard.head(10).to_markdown(index=False, numalign=\"left\", stralign=\"left\"))\n", "\n", "# Print the column names and their data types of the transformed dataframe\n", "print(df_encoded_standard.info())" ], "outputs": [], "execution_count": null, "metadata": { "id": "aMd1Lqcbc2JD" } }, { "cell_type": "code", "source": [ "from sklearn.preprocessing import MinMaxScaler # Importing MinMaxScaler here\n", "\n", "# Define the numerical features\n", "numerical_features = ['Age', 'Salary']\n", "\n", "# Apply MinMaxScaler to the numerical features\n", "min_max_scaler = MinMaxScaler()\n", "df_encoded_min_max = df.copy()\n", "df_encoded_min_max[numerical_features] = min_max_scaler.fit_transform(df_encoded_min_max[numerical_features])\n", "\n", "# Print the first 5 rows of the dataframe\n", "print(df_encoded_min_max.head().to_markdown(index=False, numalign=\"left\", stralign=\"left\"))\n", "\n", "# Print the column names and their data types of the transformed dataframe\n", "print(df_encoded_min_max.info())" ], "metadata": { "id": "0D4aESwYXd7w" }, "execution_count": null, "outputs": [] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 }