Fine_Tuned_Bert_Model_July_2024_.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ArjunNPatel/finbertuconn2024/blob/main/Fine_Tuned_Bert_Model_July_2024_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BCeFoefbbFKb"
      },
      "outputs": [],
      "source": [
        "!pip install -U transformers --quiet\n",
        "#!pip install tensorflow --quiet\n",
        "!pip install evaluate --quiet\n",
        "!pip install torch --quiet\n",
        "!pip install huggingface_hub --quiet\n",
        "!pip install tqdm boto3 requests regex sentencepiece sacremoses --quiet\n",
        "from huggingface_hub import notebook_login\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import BertTokenizer, AutoModel, AutoModelForSequenceClassification, TrainingArguments, Trainer, PreTrainedModel, PretrainedConfig\n",
        "import evaluate\n",
        "import pathlib, os, json\n",
        "#import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import tqdm, boto3, requests, regex, sentencepiece, sacremoses\n",
        "\n",
        "notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OtATRedHbsFF"
      },
      "outputs": [],
      "source": [
        "#the fine tuning corpus\n",
        "import urllib, json\n",
        "url = \"https://raw.githubusercontent.com/stocks-predictor/bert/master/BNEWS_DATA/datasetEconomyNews.json\"\n",
        "response = urllib.request.urlopen(url)\n",
        "rawdata = json.loads(response.read())\n",
        "\n",
        "data = [[\"text,label\"]]\n",
        "\n",
        "examplenum = 300\n",
        "exampletypes = [0,0,0]\n",
        "for article in rawdata:\n",
        "  splitter = [article[\"headlineTitle\"] + \" \" + article[\"headlineText\"], article[\"classification\"]]\n",
        "  if(exampletypes[splitter[1]] < examplenum/3):\n",
        "    exampletypes[splitter[1]] += 1\n",
        "  else:\n",
        "    continue\n",
        "  splitter[1] += 1\n",
        "  data.append(splitter)\n",
        "\n",
        "\n",
        "df = pd.DataFrame(data[1:], columns = data[0][0].split(\",\"))\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "%cd /content/gdrive/MyDrive/Machine_Learning_Finance_UCONN_Stamford_2024/Models\n",
        "df = df.sample(frac = 1).reset_index(drop=True)\n",
        "df1 = df.iloc[0:int(len(df)*2/3)]\n",
        "df2 = df.iloc[int(len(df)*2/3):]\n",
        "df1 = df1.sample(frac = 1).reset_index(drop=True)\n",
        "df2 = df2.reset_index(drop=True)\n",
        "df1.to_csv(r\"myfinetuningdataset_train.csv\")\n",
        "df1['label'].value_counts().plot(kind='bar')\n",
        "df2.to_csv(r\"myfinetuningdataset_test.csv\")\n",
        "df2['label'].value_counts().plot(kind='bar')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uNpBsw31kogQ"
      },
      "outputs": [],
      "source": [
        "!pip install datasets --quiet\n",
        "from datasets import load_dataset\n",
        "\n",
        "tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'tokenizer', 'yiyanghkust/finbert-tone')\n",
        "def tokenize_function(examples):\n",
        "     return tokenizer(examples[\"text\"], padding = \"max_length\", truncation=True, max_length=128, return_tensors = \"pt\")\n",
        "\n",
        "dataset = load_dataset('csv', data_files= {'train': 'myfinetuningdataset_train.csv', 'test': 'myfinetuningdataset_test.csv' })\n",
        "dataset = dataset.remove_columns([\"Unnamed: 0\"])\n",
        "\n",
        "tokenized_datasets = dataset.map(tokenize_function, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y5wooSC1sB0-"
      },
      "outputs": [],
      "source": [
        "from huggingface_hub import PyTorchModelHubMixin\n",
        "! pip install -U 'accelerate>=0.21.0' --quiet\n",
        "import accelerate\n",
        "accelerate.__version__\n",
        "\n",
        "\n",
        "\n",
        "class BertForSequenceClassification( nn.Module, PyTorchModelHubMixin):\n",
        "    def __init__(self, pretrained_model_name, num_labels=3):\n",
        "        super(BertForSequenceClassification, self).__init__()\n",
        "        self.num_labels = num_labels\n",
        "        self.bert = torch.hub.load('huggingface/pytorch-transformers', 'model', pretrained_model_name)\n",
        "        self.tokenizer = tokenizer\n",
        "        self.loss_fn = nn.CrossEntropyLoss()\n",
        "        self.dropout = nn.Dropout()\n",
        "        self.fc1 = nn.Linear(self.bert.config.hidden_size, 128)\n",
        "        self.finaloutput = nn.Linear(128, num_labels)\n",
        "        self.softmaxlayer = nn.Softmax(dim=1)\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, labels = None, *args, **kwargs):\n",
        "        #print(kwargs)\n",
        "        #print(args)\n",
        "        outputs = self.bert(input_ids = input_ids, attention_mask = attention_mask)\n",
        "        pooled_output = outputs.pooler_output\n",
        "        pooled_output = self.dropout(pooled_output)\n",
        "        fc1_output = nn.functional.relu(self.fc1(pooled_output))\n",
        "        logits = self.finaloutput(fc1_output)\n",
        "        logits = self.softmaxlayer(logits)\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "          loss = self.loss_fn(logits.view(-1, self.num_labels), labels.view(-1))\n",
        "        return {\"logits\":logits,\n",
        "                \"loss\": loss\n",
        "                }\n",
        "\n",
        "pretrained_model_name = \"yiyanghkust/finbert-pretrain\"\n",
        "the_model_pytorch = BertForSequenceClassification(pretrained_model_name)\n",
        "# Print model summary\n",
        "#print(the_model_pytorch)\n",
        "metric = evaluate.load(\"accuracy\")\n",
        "def compute_metrics(eval_pred):\n",
        "    logits = eval_pred.predictions\n",
        "    labels = eval_pred.label_ids\n",
        "    #print(logits)\n",
        "    #print(labels)\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return metric.compute(predictions=predictions, references=labels)\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"test_trainer\",\n",
        "    #remove_unused_columns=False,\n",
        "    accelerator_config  = {\"use_seedable_sampler\": False},\n",
        "    num_train_epochs=5,\n",
        "    eval_strategy = 'epoch'\n",
        "    #load_best_model_at_end = True\n",
        "    )\n",
        "\n",
        "trainer = Trainer(\n",
        "    model = the_model_pytorch,\n",
        "    args=training_args,\n",
        "    train_dataset= tokenized_datasets[\"train\"],\n",
        "    eval_dataset= tokenized_datasets[\"test\"],\n",
        "    compute_metrics=compute_metrics,\n",
        ")\n",
        "\n",
        "trainer.train()\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\"\"\"\n",
        "the_model_pytorch.push_to_hub(\"finbert-tone-v0\")\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "eBn_s5qJqTYt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dr8ocnuZlA-8"
      },
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}