scrape_new.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5e3b37bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from requests.packages.urllib3.exceptions import InsecureRequestWarning\n",
    "import re\n",
    "import nltk\n",
    "import ssl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "39cc09dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    _create_unverified_https_context = ssl._create_unverified_context\n",
    "except AttributeError:\n",
    "    pass\n",
    "else:\n",
    "    ssl._create_default_https_context = _create_unverified_https_context\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop = stopwords.words('english')\n",
    "\n",
    "requests.packages.urllib3.disable_warnings(InsecureRequestWarning)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "d728a09e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_phone_numbers(string):\n",
    "    r = re.compile(r'(\\d{4}[-\\.\\s]??\\d{4}[-\\.\\s]??\\d{4}|\\(\\d{}\\)\\s*\\d{4}[-\\.\\s]??\\d{4}|\\d{4}[-\\.\\s]??\\d{4})')\n",
    "    phone_numbers = r.findall(string)\n",
    "    return phone_numbers\n",
    "\n",
    "def extract_email_addresses(string):\n",
    "    string = string.replace(\"[at]\",\"@\")\n",
    "    r = re.compile(r'[\\w\\.-]+@[\\w\\.-]+')\n",
    "    return r.findall(string)\n",
    "\n",
    "\n",
    "def ie_preprocess(document):\n",
    "    document = ' '.join([i for i in document.split() if i not in stop])\n",
    "    sentences = nltk.sent_tokenize(document)\n",
    "    sentences = [nltk.word_tokenize(sent) for sent in sentences]\n",
    "    sentences = [nltk.pos_tag(sent) for sent in sentences]\n",
    "    return sentences\n",
    "\n",
    "def extract_names(document):\n",
    "    names = []\n",
    "    sentences = ie_preprocess(document)\n",
    "    for tagged_sentence in sentences:\n",
    "        for chunk in nltk.ne_chunk(tagged_sentence):\n",
    "            if type(chunk) == nltk.tree.Tree:\n",
    "                if chunk.label() == 'PERSON':\n",
    "                    names.append(' '.join([c for c in chunk]))\n",
    "    return names, extract_email_addresses(document), extract_phone_numbers(document)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ba409b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.iitb.ac.in/en/education/academic-divisions\"\n",
    "\n",
    "r = requests.get(url, verify=False)\n",
    "htmlContent = r.content\n",
    "soup = BeautifulSoup(htmlContent, \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "23eae8b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Biosciences and Bioengineering-------------------------------+=+++++++++++++\n",
      "(['Faculty', 'Facebook', 'Twitter Biosciences BioengineeringIndian Institute Technology Bombay Home', 'Areas Facilities', 'Bombay Toggle Menu Faculty Members Balaji', 'Kanekar', 'Kumar', 'Patankar', 'Patkar', 'Paul', 'Punekar', 'Adjunct Faculty Soni', 'Faculty Banerjee', 'Us Department Biosciences BioengineeringIndian Institute Technology BombayPowai', 'Facebook Twitter', 'Seminars Seminar Room Booking Useful Forms', 'Moodle Institute Calendar Quick', 'Calendar Commemorating Prof Rinti Banerjee', 'Bombay Scroll', 'Areas Facilities'], ['balaji@iitb.ac.inPhone', 'abanerjee@iitb.ac.inPhone', 'pbhaumik@iitb.ac.inPhone', 'santanughosh@iitb.ac.inPhone', 'sandipk@iitb.ac.inPhone', 'nkanekar@iitb.ac.inPhone', 'kirankondabagil@iitb.ac.inPhone', 'ashutoshk@iitb.ac.inPhone', 'akunwar@iitb.ac.inPhone', 'samirmaji@iitb.ac.inPhone', 'rmallik@iitb.ac.inPhone', 'rmanch@iitb.ac.inPhone', 'mukherji@iitb.ac.inPhone', 'sreelaja@iitb.ac.inPhone', 'ranjithp@iitb.ac.inPhone', 'panda@iitb.ac.inPhone', 'patankar@iitb.ac.inPhone', 'rajeshpatkar@iitb.ac.inPhone', 'debjani.paul@iitb.ac.inPhone', 'pphale@iitb.ac.inPhone', 'nsp@iitb.ac.inPhone', 'purwarrahul@iitb.ac.inPhone', 'shamiks@iitb.ac.inPhone', 'rsrivasta@iitb.ac.inPhone', 'sanjeeva@iitb.ac.inPhone', 'gsm@iitb.ac.inPhone', 'prakriti@iitb.ac.inPhone', 'harivarma@iitb.ac.inPhone', 'profvps@yahoo.comPhone', 'rvhosur53@yahoo.comPhone', 'kkr@iitb.ac.in', 'jayadeva@iitb.ac.in', 'office.bio@iitb.ac.in'], ['2576-7778', '2576-7794', '2576-7748', '2576-7766', '2576-7706', '2576-7735', '2576-7758', '2576-7762', '2576-7799', '2576-7774', '2576-7769', '2576-7765', '2576-7767', '2576-6758', '2576-7761', '2576-7838', '2576-7773', '2576-4768', '2576-7798', '2576-7836', '2576-7775', '2576-7737', '2576-7743', '2576-7746', '2576-7779', '2576-7777', '2576-7797', '2576-7705', '2576-4768', '2576-4768', '2576 7771'])\n"
     ]
    }
   ],
   "source": [
    "departments = ['Biosciences and Bioengineering']\n",
    "print(\"\\n\")\n",
    "for dept_name in departments:\n",
    "    print(dept_name+\"-------------------------------+=+++++++++++++\")\n",
    "    try:\n",
    "        for list in soup.findAll('a'):\n",
    "            if list.text.strip() == dept_name:\n",
    "                d = str(list.get('href'))\n",
    "                dept = requests.get(d,verify=False)\n",
    "                deptContent = dept.content\n",
    "                soup_dept = BeautifulSoup(deptContent,\"html.parser\")\n",
    "\n",
    "        for list in soup_dept.findAll('a'):\n",
    "            if list.text.strip() == \"Faculty\":\n",
    "                f = list.get('href')\n",
    "                try:\n",
    "                    faculty = requests.get(f,verify=False)\n",
    "                except:\n",
    "                    print(d+f)\n",
    "                    faculty = requests.get(d+f, verify=False)\n",
    "                facultyContent = faculty.content\n",
    "                soup_faculty = BeautifulSoup(facultyContent, \"html.parser\")\n",
    "\n",
    "        table_body = soup_faculty.find('tbody')\n",
    "        data=[]\n",
    "        rows = table_body.find_all('tr')\n",
    "        for row in rows:\n",
    "            cols = row.find_all('td')\n",
    "            cols = [ele.text.strip() for ele in cols]\n",
    "            data.append([ele for ele in cols if ele])\n",
    "\n",
    "        for i in data:\n",
    "            for j in i:\n",
    "                print(j) \n",
    "    except:\n",
    "        names = extract_names(str(soup_faculty.text.strip()))\n",
    "        print(names)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a44b5a4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a97e1aa6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}