clean_previous_seasons.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/saheedniyi02/fpl-ai/blob/master/clean_previous_seasons.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cUidC2SgUA7c"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "#from config import *\n",
        "#from utils import get_all_players_last_stats,create_features\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aHk-ncULU1fS"
      },
      "outputs": [],
      "source": [
        "seasons=[\"2020-21\",\"2021-22\",\"2022-23\"]\n",
        "\n",
        "unused_columns=[\"xP\",\"opponent_team\",\"expected_assists\",\"expected_goal_involvements\",\"expected_goals\",\n",
        "                \"expected_goals_conceded\",\"element_type_ex\",\"team_h_score\",\"team_a_score\",\"element\" , \"round\" ,\"fixture\",\"starts\" ]\n",
        "\n",
        "#stats that you want to use their values for the last few games to predict a\n",
        "\n",
        "no_last_stats=3 #number of last gameweek stats to use for prediction\n",
        "\n",
        "history_stats=[\"assists\",\"bonus\",\"bps\",\"clean_sheets\",\"creativity\",\"goals_conceded\",\"goals_scored\",\"ict_index\",\"influence\",\"minutes\",\n",
        "               \"own_goals\",\"penalties_missed\",\"penalties_saved\",\"red_cards\",\"saves\",\"selected\",\"threat\",\"total_points\",\"transfers_in\",\"transfers_out\",\"value\",\"match_result\",\"team Goal scored\",\"team Goal conceded\"]\n",
        "\n",
        "last_features=[f\"last {no_last_stats} {stat}\" for stat in history_stats]\n",
        "\n",
        "mean_features=[\"assists\",\"bonus\",\"bps\",\"clean_sheets\",\"creativity\",\"goals_conceded\",\n",
        "     \"goals_scored\",\"ict_index\",\"influence\",\"minutes\"\n",
        "     ,\"own_goals\",\"penalties_missed\",\"penalties_saved\",\n",
        "     \"red_cards\",\"saves\",\"selected\",\"threat\",\"total_points\"\n",
        "     ,\"transfers_in\",\"transfers_out\",\"value\",\"match_result\",\"team Goal scored\",\"team Goal conceded\"]\n",
        "\n",
        "std_features=[\"bps\",\"creativity\",\"ict_index\",\"influence\",\"minutes\",\"selected\",\"threat\",\"total_points\",\"transfers_in\",\"transfers_out\",\"value\"]\n",
        "\n",
        "\n",
        "\n",
        "dropped_columns = [\n",
        "    \"season\",\n",
        "    \"opponent\",\n",
        "    \"match_result\",\n",
        "    \"position\",\n",
        "    \"assists\",\n",
        "    \"penalties_missed\",\n",
        "    \"bonus\",\n",
        "    \"bps\",\n",
        "    \"clean_sheets\",\n",
        "    \"creativity\",\n",
        "    \"goals_conceded\",\n",
        "    \"goals_scored\",\n",
        "    \"ict_index\",\n",
        "    \"influence\",\n",
        "    \"own_goals\",\n",
        "    \"penalties_saved\",\n",
        "    \"red_cards\",\n",
        "    \"saves\",\n",
        "    \"selected\",\n",
        "    \"threat\",\n",
        "    \"transfers_balance\",\n",
        "    \"transfers_in\",\n",
        "    \"transfers_out\",\n",
        "    \"yellow_cards\"\n",
        "]  # \"value\",\n",
        "\n",
        "\n",
        "\n",
        "leak_columns = [\n",
        "    \"name\",\n",
        "    \"team\",\n",
        "]  # columns that shouldnt be used in training fir fear of data leakage\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "team_features=[\"team Goal scored\",\"team Goal conceded\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5hxfXhIXVIEN"
      },
      "outputs": [],
      "source": [
        "def get_last_stats(data, stat, name, no_last_stats=3):\n",
        "    name_df = data[data[\"name\"] == name]\n",
        "    seasons = [\"2020-21\", \"2021-22\", \"2022-23\"]\n",
        "    name_df_dict = {}\n",
        "    for season in seasons:\n",
        "        name_df_season = name_df[name_df[\"season\"] == season]\n",
        "        list_stats = []\n",
        "        stats_x = []\n",
        "        for value in name_df_season[stat]:\n",
        "            if len(stats_x) < no_last_stats:\n",
        "                list_stats.append(np.array(stats_x))\n",
        "                stats_x.append(value)\n",
        "            else:\n",
        "                list_stats.append(np.array(stats_x))\n",
        "                stats_x = deque_and_queue(stats_x, value)\n",
        "        name_df_season[f\"last {no_last_stats} {stat}\"] = list_stats\n",
        "        name_df_dict[season] = name_df_season\n",
        "    return pd.concat(name_df_dict.values())\n",
        "\n",
        "\n",
        "def get_all_players_last_stats(data, stat, no_last_stats=3):\n",
        "    players_df = []\n",
        "    for player in data[\"name\"].unique():\n",
        "        data_player = get_last_stats(data, stat, player, no_last_stats)\n",
        "        players_df.append(data_player)\n",
        "    # print(pd.concat(players_df))\n",
        "    return pd.concat(players_df)\n",
        "\n",
        "\n",
        "\n",
        "def remove_neg(val):\n",
        "    if val > 0:\n",
        "        return val\n",
        "    else:\n",
        "        return -val\n",
        "\n",
        "\n",
        "def deque_and_queue(stats, value):\n",
        "    # if -1 in stats:\n",
        "    #  return stats\n",
        "    # deque\n",
        "    stats = stats[1:]\n",
        "    stats.append(value)\n",
        "    return stats\n",
        "\n",
        "\n",
        "def convert_minutes(val):\n",
        "    \"\"\"CONVERTS MINUTES TO A CATEGORICAL OUTPUT\"\"\"\n",
        "    if val > 10:\n",
        "        return 1\n",
        "    else:\n",
        "        return 0\n",
        "\n",
        "\n",
        "def find_mode(vals):\n",
        "    \"\"\"find the mode of vals\"\"\"\n",
        "    try:\n",
        "        if -1 in vals:\n",
        "            return -1\n",
        "        return stats.mode(vals)[0][0]\n",
        "    except IndexError:\n",
        "        return np.nan\n",
        "\n",
        "\n",
        "def find_mean(vals):\n",
        "    \"\"\"find the mean of vals\"\"\"\n",
        "    try:\n",
        "        if -1 in vals:\n",
        "            return -1\n",
        "        return np.mean(vals)\n",
        "    except:\n",
        "        return np.nan\n",
        "\n",
        "\n",
        "def find_max(vals):\n",
        "    \"\"\"find the maximum of vals\"\"\"\n",
        "    try:\n",
        "        if -1 in vals:\n",
        "            return -1\n",
        "        return np.max(vals)\n",
        "    except:\n",
        "        return np.nan\n",
        "\n",
        "\n",
        "def find_std(vals):\n",
        "    \"\"\"find the standard deviation of vals\"\"\"\n",
        "    try:\n",
        "        if -1 in vals:\n",
        "            return -1\n",
        "        return np.std(vals)\n",
        "    except:\n",
        "        return np.nan\n",
        "\n",
        "\n",
        "def find_value_count(vals, to_count):\n",
        "    \"\"\"find the number of times to_count appears in vals\"\"\"\n",
        "    try:\n",
        "        if -1 in vals:\n",
        "            return -1\n",
        "        values, count = np.unique(vals, return_counts=True)\n",
        "        index = np.where(values == to_count)[0][0]\n",
        "        return count[index]\n",
        "\n",
        "    except:\n",
        "        return -2\n",
        "\n",
        "def get_all_stats(data, stat, name):\n",
        "    name_df = data[data[\"name\"] == name]\n",
        "    seasons = [\"2020-21\", \"2021-22\", \"2022-23\"]\n",
        "    name_df_dict = {}\n",
        "    for season in seasons:\n",
        "        name_df_season = name_df[name_df[\"season\"] == season]\n",
        "        list_stats = []\n",
        "        stats_x = []\n",
        "        for value in name_df_season[stat]:\n",
        "          list_stats.append(np.array(stats_x))\n",
        "          stats_x.append(value)\n",
        "\n",
        "\n",
        "        name_df_season[f\"all {stat}\"] = list_stats\n",
        "        name_df_dict[season] = name_df_season\n",
        "    return pd.concat(name_df_dict.values())\n",
        "\n",
        "\n",
        "def get_all_players_all_stats(data, stat):\n",
        "    players_df = []\n",
        "    for player in data[\"name\"].unique():\n",
        "        data_player =  get_all_stats(data, stat, player)\n",
        "        players_df.append(data_player)\n",
        "    # print(pd.concat(players_df))\n",
        "    return pd.concat(players_df)\n",
        "\n",
        "def create_features(data, mean_features, std_features):\n",
        "    for mean_feature in mean_features:\n",
        "        print(mean_feature)\n",
        "        data[f\"mean {mean_feature} {no_last_stats}\"] = [\n",
        "            find_mean(values) for values in data[f\"last {no_last_stats} {mean_feature}\"]\n",
        "        ]\n",
        "        data[f\"mean {mean_feature} all\"] = [\n",
        "            find_mean(values) for values in data[f\"all {mean_feature}\"]\n",
        "        ]\n",
        "        #data[f\"mean {mean_feature} all\"]= data[f\"mean {mean_feature} all\"]/data[\"GW\"]\n",
        "    for std_feature in std_features:\n",
        "        print(std_feature)\n",
        "        data[f\"std {std_feature} {no_last_stats}\"] = [\n",
        "            find_std(values) for values in data[f\"last {no_last_stats} {std_feature}\"]\n",
        "        ]\n",
        "    return data\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l8m5Jl5ZoizX",
        "outputId": "5d15c0fc-d898-43a8-c04b-7931fbcd974d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(76469, 54)\n",
            "(76317, 54)\n",
            "assists\n",
            "bonus\n",
            "bps\n",
            "clean_sheets\n",
            "creativity\n",
            "goals_conceded\n",
            "goals_scored\n",
            "ict_index\n",
            "influence\n",
            "minutes\n",
            "own_goals\n",
            "penalties_missed\n",
            "penalties_saved\n",
            "red_cards\n",
            "saves\n",
            "selected\n",
            "threat\n",
            "total_points\n",
            "transfers_in\n",
            "transfers_out\n",
            "value\n",
            "match_result\n",
            "team Goal scored\n",
            "team Goal conceded\n",
            "assists\n",
            "bonus\n",
            "bps\n",
            "clean_sheets\n",
            "creativity\n",
            "goals_conceded\n",
            "goals_scored\n",
            "ict_index\n",
            "influence\n",
            "minutes\n",
            "own_goals\n",
            "penalties_missed\n",
            "penalties_saved\n",
            "red_cards\n",
            "saves\n",
            "selected\n",
            "threat\n",
            "total_points\n",
            "transfers_in\n",
            "transfers_out\n",
            "value\n",
            "match_result\n",
            "team Goal scored\n",
            "team Goal conceded\n",
            "bps\n",
            "creativity\n",
            "ict_index\n",
            "influence\n",
            "minutes\n",
            "selected\n",
            "threat\n",
            "total_points\n",
            "transfers_in\n",
            "transfers_out\n",
            "value\n"
          ]
        }
      ],
      "source": [
        "data = pd.read_csv(\"/content/drive/MyDrive/Fplpredict /datasets/previous_seasons.csv\", index_col=0)\n",
        "data_home=data[data[\"was_home\"]==True]\n",
        "data_away=data[data[\"was_home\"]==False]\n",
        "data_home[\"team Goal scored\"]=data_home[\"team_h_score\"]\n",
        "data_home[\"team Goal conceded\"]=data_home[\"team_a_score\"]\n",
        "data_away[\"team Goal conceded\"]=data_away[\"team_h_score\"]\n",
        "data_away[\"team Goal scored\"]=data_away[\"team_a_score\"]\n",
        "data=pd.concat([data_home,data_away])\n",
        "#data[\"Goal_difference\"]=data[\"team_h_score\"]-data[\"team_a_score\"]\n",
        "data.drop(unused_columns,axis=1,inplace=True)\n",
        "\n",
        "data[\"kickoff_time\"]=pd.to_datetime(data[\"kickoff_time\"])\n",
        "print(data.shape)\n",
        "\n",
        "data.drop_duplicates([\"name\",\"GW\",\"season\",\"team\",\"opponent\"],inplace=True)\n",
        "print(data.shape)\n",
        "data.sort_values(by=\"kickoff_time\",inplace=True)\n",
        "\n",
        "\n",
        "for stat in history_stats:\n",
        "    print(stat)\n",
        "    data=get_all_players_last_stats(data,stat)\n",
        "    data=get_all_players_all_stats(data,stat)\n",
        "\n",
        "data=create_features(data,mean_features,std_features)\n",
        "\n",
        "data.drop([f\"last {no_last_stats} {stat}\" for stat in history_stats],axis=1,inplace=True)\n",
        "data.drop([f\"all {stat}\" for stat in history_stats],axis=1,inplace=True)\n",
        "data[\"ratio_goal_scored 3\"]=((data[\"mean goals_scored 3\"]/data[\"mean team Goal scored 3\"]).replace(np.inf, 0))*100\n",
        "data[\"ratio_goal_scored all\"]=((data[\"mean goals_scored all\"]/data[\"mean team Goal scored all\"]).replace(np.inf, 0))*100"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oxBobE1GUN4i",
        "outputId": "3704fdc1-7898-4654-9bfd-554b3aff8cab"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "396          Arsenal\n",
              "503            Leeds\n",
              "518      Aston Villa\n",
              "524           Wolves\n",
              "543    Sheffield Utd\n",
              "           ...      \n",
              "39           Chelsea\n",
              "40          Brighton\n",
              "33         Brentford\n",
              "762            Leeds\n",
              "530        Liverpool\n",
              "Name: opponent, Length: 76317, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "data[\"opponent\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ntp-9VWyNGwQ"
      },
      "outputs": [],
      "source": [
        "#add opponent stats for gameweeks\n",
        "list_data_gw=[]\n",
        "for season in seasons:\n",
        "  for GW in range(1,39):\n",
        "\n",
        "    data_gw=data[(data[\"season\"]==season) & (data[\"GW\"]==GW)]\n",
        "    data_opponent=data_gw.drop_duplicates(subset=[\"kickoff_time\",\"team\"],keep=\"first\")[['team','mean team Goal scored 3','mean team Goal conceded 3','mean team Goal scored all','mean team Goal conceded all',\"mean match_result all\"]]\n",
        "    #print(data_gw.shape)\n",
        "    data_opponent.columns=['opponent','opp mean team Goal scored 3','opp mean team Goal conceded 3','opp mean team Goal scored all','opp mean team Goal conceded all',\"opp mean match_result all\"]\n",
        "    #print(data_opponent.shape)\n",
        "    data_gw=pd.merge(data_gw,data_opponent,on=\"opponent\",how=\"left\")\n",
        "    list_data_gw.append(data_gw)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(data.columns)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Ix9dFQGCYZd",
        "outputId": "43ac5606-35d0-48b4-afff-88a509dfbb3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['name',\n",
              " 'position',\n",
              " 'team',\n",
              " 'assists',\n",
              " 'bonus',\n",
              " 'bps',\n",
              " 'clean_sheets',\n",
              " 'creativity',\n",
              " 'goals_conceded',\n",
              " 'goals_scored',\n",
              " 'ict_index',\n",
              " 'influence',\n",
              " 'kickoff_time',\n",
              " 'minutes',\n",
              " 'own_goals',\n",
              " 'penalties_missed',\n",
              " 'penalties_saved',\n",
              " 'red_cards',\n",
              " 'saves',\n",
              " 'selected',\n",
              " 'threat',\n",
              " 'total_points',\n",
              " 'transfers_balance',\n",
              " 'transfers_in',\n",
              " 'transfers_out',\n",
              " 'value',\n",
              " 'was_home',\n",
              " 'yellow_cards',\n",
              " 'last_season_position',\n",
              " 'percent_value',\n",
              " 'position rank',\n",
              " 'match_result',\n",
              " 'goals_scored_ex',\n",
              " 'assists_ex',\n",
              " 'total_points_ex',\n",
              " 'minutes_ex',\n",
              " 'goals_conceded_ex',\n",
              " 'creativity_ex',\n",
              " 'influence_ex',\n",
              " 'threat_ex',\n",
              " 'bonus_ex',\n",
              " 'bps_ex',\n",
              " 'ict_index_ex',\n",
              " 'clean_sheets_ex',\n",
              " 'red_cards_ex',\n",
              " 'yellow_cards_ex',\n",
              " 'selected_by_percent_ex',\n",
              " 'now_cost_ex',\n",
              " 'season',\n",
              " 'GW',\n",
              " 'opponent',\n",
              " 'opponent_last_season_position',\n",
              " 'team Goal scored',\n",
              " 'team Goal conceded',\n",
              " 'mean assists 3',\n",
              " 'mean assists all',\n",
              " 'mean bonus 3',\n",
              " 'mean bonus all',\n",
              " 'mean bps 3',\n",
              " 'mean bps all',\n",
              " 'mean clean_sheets 3',\n",
              " 'mean clean_sheets all',\n",
              " 'mean creativity 3',\n",
              " 'mean creativity all',\n",
              " 'mean goals_conceded 3',\n",
              " 'mean goals_conceded all',\n",
              " 'mean goals_scored 3',\n",
              " 'mean goals_scored all',\n",
              " 'mean ict_index 3',\n",
              " 'mean ict_index all',\n",
              " 'mean influence 3',\n",
              " 'mean influence all',\n",
              " 'mean minutes 3',\n",
              " 'mean minutes all',\n",
              " 'mean own_goals 3',\n",
              " 'mean own_goals all',\n",
              " 'mean penalties_missed 3',\n",
              " 'mean penalties_missed all',\n",
              " 'mean penalties_saved 3',\n",
              " 'mean penalties_saved all',\n",
              " 'mean red_cards 3',\n",
              " 'mean red_cards all',\n",
              " 'mean saves 3',\n",
              " 'mean saves all',\n",
              " 'mean selected 3',\n",
              " 'mean selected all',\n",
              " 'mean threat 3',\n",
              " 'mean threat all',\n",
              " 'mean total_points 3',\n",
              " 'mean total_points all',\n",
              " 'mean transfers_in 3',\n",
              " 'mean transfers_in all',\n",
              " 'mean transfers_out 3',\n",
              " 'mean transfers_out all',\n",
              " 'mean value 3',\n",
              " 'mean value all',\n",
              " 'mean match_result 3',\n",
              " 'mean match_result all',\n",
              " 'mean team Goal scored 3',\n",
              " 'mean team Goal scored all',\n",
              " 'mean team Goal conceded 3',\n",
              " 'mean team Goal conceded all',\n",
              " 'std bps 3',\n",
              " 'std creativity 3',\n",
              " 'std ict_index 3',\n",
              " 'std influence 3',\n",
              " 'std minutes 3',\n",
              " 'std selected 3',\n",
              " 'std threat 3',\n",
              " 'std total_points 3',\n",
              " 'std transfers_in 3',\n",
              " 'std transfers_out 3',\n",
              " 'std value 3',\n",
              " 'ratio_goal_scored 3',\n",
              " 'ratio_goal_scored all']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "soB0XsTZlOwU"
      },
      "outputs": [],
      "source": [
        "data=pd.concat(list_data_gw)\n",
        "data.to_csv(\"/content/drive/MyDrive/Fplpredict /cleaned_dataset/cleaned_previous_seasons.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s5PL85TbCrAM",
        "outputId": "7c5790a0-70c2-4484-813c-afb01a6ac3b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(86852, 120)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[pd.isna(data[\"opp mean team Goal scored all\"])][\"GW\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i5ztpPw0-IeL",
        "outputId": "5c3aaefb-24a6-4141-d08c-64ff29529d3d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1     1552\n",
              "2       80\n",
              "21      67\n",
              "9       32\n",
              "5       30\n",
              "20      29\n",
              "6       28\n",
              "Name: GW, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "12U8magIDgpobeD-sbHj3HixIJrvrRqWi",
      "authorship_tag": "ABX9TyO/7pL04gExUM4Koj2XI4Bt",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}