remove pandas and refactor

Author: cassiofb-dev

.tool-versions

@@ -1 +1 @@
-python 3.9.14
+python 3.10.7

README.md

@@ -6,6 +6,7 @@
 
 <p align="center">
   <a href="#about">About</a> •
+  <a href="#features">Features</a> •
   <a href="#usage">Usage</a> •
   <a href="#credits">Credits</a> •
   <a href="#license">License</a>
@@ -17,6 +18,12 @@
 
 Working with FIDE oficial data is not simple, mainly because they don't have an API. That's the reason I made a simple API with FastAPI to scrape the data from their own website and provide it as JSON over HTTP requests.
 
+## Features
+
+- Get top players list
+- Get player info
+- Get player history
+
 ## Usage
 
 You will need git and asdf installed, from your terminal:
@@ -26,22 +33,24 @@ git clone https://github.com/cassiofb-dev/fide-api
 
 cd fide-api
 
-asdf install python 3.9.14
+asdf install python 3.10.7
 
-asdf local python 3.9.14
+asdf local python 3.10.7
 
 python -m venv venv
 
 source venv/bin/activate
 
 pip install -r requirements.txt
 
-uvicorn main:app --reload
+uvicorn src.api:app --reload
 ```
 
+To see the docs go to ``localhost:8000/docs``
+
 ## Credits
 
-This project uses git, python (3.9.14) and asdf.
+This project uses git, python (3.10.7) and asdf.
 
 The following python dependecies were used:
 ```txt
@@ -50,31 +59,19 @@ beautifulsoup4==4.11.1
 certifi==2022.9.24
 charset-normalizer==2.1.1
 click==8.1.3
-dnspython==2.2.1
-email-validator==1.3.0
 fastapi==0.85.0
 h11==0.14.0
 httptools==0.5.0
 idna==3.4
-itsdangerous==2.1.2
-Jinja2==3.1.2
-MarkupSafe==2.1.1
-numpy==1.23.3
 orjson==3.8.0
-pandas==1.5.0
 pydantic==1.10.2
-python-dateutil==2.8.2
 python-dotenv==0.21.0
-python-multipart==0.0.5
-pytz==2022.4
 PyYAML==6.0
 requests==2.28.1
-six==1.16.0
 sniffio==1.3.0
 soupsieve==2.3.2.post1
 starlette==0.20.4
-typing_extensions==4.3.0
-ujson==5.5.0
+typing_extensions==4.4.0
 urllib3==1.26.12
 uvicorn==0.18.3
 uvloop==0.17.0

requirements.txt

@@ -3,31 +3,19 @@ beautifulsoup4==4.11.1
 certifi==2022.9.24
 charset-normalizer==2.1.1
 click==8.1.3
-dnspython==2.2.1
-email-validator==1.3.0
 fastapi==0.85.0
 h11==0.14.0
 httptools==0.5.0
 idna==3.4
-itsdangerous==2.1.2
-Jinja2==3.1.2
-MarkupSafe==2.1.1
-numpy==1.23.3
 orjson==3.8.0
-pandas==1.5.0
 pydantic==1.10.2
-python-dateutil==2.8.2
 python-dotenv==0.21.0
-python-multipart==0.0.5
-pytz==2022.4
 PyYAML==6.0
 requests==2.28.1
-six==1.16.0
 sniffio==1.3.0
 soupsieve==2.3.2.post1
 starlette==0.20.4
-typing_extensions==4.3.0
-ujson==5.5.0
+typing_extensions==4.4.0
 urllib3==1.26.12
 uvicorn==0.18.3
 uvloop==0.17.0

scraper/fide_scraper.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from fastapi.responses import ORJSONResponse
 
-from scraper import fide_scraper
+from src.scraper import fide_scraper
 
 app = FastAPI(default_response_class=ORJSONResponse)
 
@@ -15,15 +15,15 @@ def home():
 
 @app.get("/top_players/")
 async def top_players(limit: int = 100, history: bool = False):
-  response = fide_scraper.top_players(limit=limit, history=history)
+  response = fide_scraper.get_top_players(limit=limit, history=history)
   return response
 
 @app.get("/player_history/")
 async def player_history(fide_id: str):
-  response = fide_scraper.player_history(fide_id=fide_id)
+  response = fide_scraper.get_player_history(fide_id=fide_id)
   return response
 
 @app.get("/player_info/")
 async def player_info(fide_id: str, history: bool = False):
-  response = fide_scraper.player_info(fide_id=fide_id, history=history)
+  response = fide_scraper.get_player_info(fide_id=fide_id, history=history)
   return response

scraper/functions/__init__.py

@@ -0,0 +1,54 @@
+import requests
+import src.scraper.functions as scraper
+
+def get_top_players(limit: int = 100, history: bool = False) -> list[dict]:
+  fide_top_players_page = requests.get("https://ratings.fide.com/top.phtml")
+
+  html_doc = fide_top_players_page.text
+
+  top_players = scraper.get_top_players(html_doc)
+
+  top_players = top_players[0:limit]
+
+  if history == False: return top_players
+
+  for player_dict in top_players:
+    fide_profile_page = f"https://ratings.fide.com/profile/{player_dict['fide_id']}"
+
+    response = requests.get(fide_profile_page)
+
+    html_doc = response.text
+
+    player_history = scraper.get_player_history(html_doc)
+
+    player_dict["history"] = player_history
+
+  return top_players
+
+def get_player_history(fide_id: str) -> list[dict]:
+  fide_profile_page = f"https://ratings.fide.com/profile/{fide_id}"
+
+  response = requests.get(fide_profile_page)
+
+  html_doc = response.text
+
+  player_history = scraper.get_player_history(html_doc)
+
+  return player_history
+
+def get_player_info(fide_id: str, history: bool = False):
+  fide_profile_page = f"https://ratings.fide.com/profile/{fide_id}"
+
+  response = requests.get(fide_profile_page)
+
+  html_doc = response.text
+
+  player_info = scraper.get_player_info(html_doc)
+
+  if history == False: return player_info
+
+  player_history = scraper.get_player_history(html_doc)
+
+  player_info["history"] = player_history
+
+  return player_info

scraper/functions/player_history.py

@@ -0,0 +1,3 @@
+from src.scraper.functions.player_history import get_player_history
+from src.scraper.functions.player_info import get_player_info
+from src.scraper.functions.top_players import get_top_players

scraper/functions/player_info.py

@@ -0,0 +1,35 @@
+from bs4 import BeautifulSoup
+
+from src.scraper.functions.utils import fide_date_to_numeric_string
+
+def get_player_history(html_doc: str):
+  soup = BeautifulSoup(html_doc, "html.parser")
+
+  table_selector = "body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div:nth-child(3) > div > div.col-lg-12.profile-tableCont > table > tbody"
+
+  table = soup.select_one(table_selector)
+
+  rows: list = table.find_all("tr")
+
+  player_history = []
+
+  for row in rows:
+    raw_row = []
+
+    for column in row.find_all("td"):
+      raw_data = column.get_text().replace(u'\xa0', '').strip()
+
+      raw_row.append(raw_data)
+
+    player_history.append({
+      "period": raw_row[0],
+      "classical_rating": int(raw_row[1] or 0),
+      "classical_games": int(raw_row[2] or 0),
+      "rapid_rating": int(raw_row[3] or 0),
+      "rapid_games": int(raw_row[4] or 0),
+      "blitz_rating": int(raw_row[5] or 0),
+      "blitz_games": int(raw_row[6] or 0),
+      "date": fide_date_to_numeric_string(raw_row[0]),
+    })
+
+  return player_history

main.py renamed to src/api.py

@@ -0,0 +1,21 @@
+from bs4 import BeautifulSoup
+
+def get_player_info(html_doc: str):
+  soup = BeautifulSoup(html_doc, "html.parser")
+
+  player_info = {
+    "fide_id": soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-12.profile-top-info > div:nth-child(1) > div:nth-child(3) > div.profile-top-info__block__row__data").get_text().strip(),
+    "fide_title": soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-12.profile-top-info > div:nth-child(2) > div:nth-child(3) > div.profile-top-info__block__row__data").get_text().strip(),
+    "federation": soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-12.profile-top-info > div:nth-child(1) > div:nth-child(2) > div.profile-top-info__block__row__data").get_text().strip(),
+    "birth_year": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-12.profile-top-info > div:nth-child(2) > div:nth-child(1) > div.profile-top-info__block__row__data").get_text().strip() or 0),
+    "sex": soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-12.profile-top-info > div:nth-child(2) > div:nth-child(2) > div.profile-top-info__block__row__data").get_text().strip(),
+    "name": soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-top.col-lg-12 > div > div.col-lg-9.profile-top__right > div > div.col-lg-8.profile-top-title").get_text().strip(),
+    "world_rank_all": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(1) > table > tbody > tr:nth-child(1) > td:nth-child(2)").get_text().strip() or 0),
+    "world_rank_active": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(1) > table > tbody > tr:nth-child(2) > td:nth-child(2)").get_text().strip() or 0),
+    "continental_rank_all": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(2)").get_text().strip() or 0),
+    "continental_rank_active": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(3) > table > tbody > tr:nth-child(2) > td:nth-child(2)").get_text().strip() or 0),
+    "national_rank_all": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(2) > table > tbody > tr:nth-child(1) > td:nth-child(2)").get_text().strip() or 0),
+    "national_rank_active": int(soup.select_one("body > section.container.section-profile > div.row.no-gutters > div.profile-bottom.col-lg-12 > div.profile-tab-containers > div.profile-tab-container.profile-tab-container_active > div:nth-child(2) > div:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(2)").get_text().strip() or 0),
+  }
+
+  return player_info