added assets for SpokenNativQA (#386)

* added assets for SpokenNativQA

* fixed init file

* updated spokenqa scripts

* GH Actions: Use v4 for upload/download artifact (#389)

* Use v4 for upload/download artifact GH Action

* Fix formatting

* Bump evaluate version

* added assets for SpokenNativQA

* fixed init file

* updated spokenqa scripts

---------

Co-authored-by: Fahim Dalvi <faimaduddin@hbku.edu.qa>

Author: firojalam

assets/ar/QA/MultiNativQA_Allam_ZeroShot.py

@@ -0,0 +1,48 @@
+import json
+import re
+
+from llmebench.datasets import MultiNativQADataset
+from llmebench.models import AzureModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": MultiNativQADataset,
+        "task": MultiNativQATask,
+        "model": AzureModel,
+        "general_args": {"test_split": "arabic_qa"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+        Please use your expertise to answer the following Arabic question. Answer in Arabic. Please provide Answer only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+        Question: {input_sample['question']}
+
+        """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["output"].strip()
+    return content

assets/ar/QA/MultiNativQA_Fanar_ZeroShot.py

@@ -0,0 +1,48 @@
+import json
+import re
+
+from llmebench.datasets import MultiNativQADataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": MultiNativQADataset,
+        "task": MultiNativQATask,
+        "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+        Please use your expertise to answer the following Arabic question. Answer in Arabic. Please provide Answer only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+        Question: {input_sample['question']}
+
+        """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["choices"][0]["message"]["content"].strip()
+    return content

assets/ar/QA/MultiNativQA_GPT4_ZeroShot.py

@@ -47,8 +47,5 @@ def post_process(response):
     content = response["choices"][0]["message"]["content"].strip()
     content = content.replace("\n", "").strip()
     if "```json" in content:
-        # content = content.replace("```json", "").replace('```', '').replace("\n}", "}")
-        # content = content.replace("{\n", "{").replace("\",\n", "\",")
-
         content = re.search(r"```json(.*)```", content).group(1)
     return json.loads(content)["answer"]

assets/ar/QA/MultiNativQA_Mistral_7b_ZeroShot.py

@@ -45,10 +45,6 @@ def prompt(input_sample):
             "role": "user",
             "content": question_prompt,
         },
-        # {
-        #     "role": "assistant",
-        #     "content": assistant_prompt,
-        # },
     ]
 
 

assets/ar/QA/SpokenNativQA_azure_asr_Allam_ZeroShot.py

@@ -0,0 +1,48 @@
+import json
+import re
+
+from llmebench.datasets import SpokenNativQADataset
+from llmebench.models import AzureModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": SpokenNativQADataset,
+        "task": MultiNativQATask,
+        "model": AzureModel,
+        "general_args": {"test_split": "arabic_qa_azure"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+        Please use your expertise to answer the following Arabic question. Answer in Arabic. Please provide Answer only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+        Question: {input_sample['question']}
+
+        """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["output"].strip()
+    return content

assets/ar/QA/SpokenNativQA_azure_asr_Fanar_ZeroShot.py

@@ -0,0 +1,48 @@
+import json
+import re
+
+from llmebench.datasets import SpokenNativQADataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": SpokenNativQADataset,
+        "task": MultiNativQATask,
+        "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa_azure"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+        Please use your expertise to answer the following Arabic question. Answer in Arabic. Please provide Answer only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+        Question: {input_sample['question']}
+
+        """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["choices"][0]["message"]["content"].strip()
+    return content

assets/ar/QA/SpokenNativQA_azure_asr_GPT4Audio_ZeroShot.py

@@ -0,0 +1,63 @@
+import json
+import re
+
+from llmebench.datasets import SpokenNativQADataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": SpokenNativQADataset,
+        "task": MultiNativQATask,
+        "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa_azure"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    base64_wav = input_sample["wav"]
+
+    question_prompt = f"""
+    Please use your expertise to answer the following Arabic question. Answer in Arabic and rate your confidence level from 1 to 10. Provide your response in the following JSON format: {{"answer": "your answer", "score": your confidence score}}. Please provide JSON output only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+    Question: {input_sample['question']}
+    
+    """
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": question_prompt,
+                },
+                {
+                    "type": "input_audio",
+                    "input_audio": {"data": base64_wav, "format": "wav"},
+                },
+            ],
+        }
+    ]
+
+
+def post_process(response):
+    content = response["choices"][0]["message"]["content"].strip()
+    content = content.replace("\n", "").strip()
+    if "```json" in content:
+        content = re.search(r"```json(.*)```", content).group(1)
+    return json.loads(content)["answer"]

assets/ar/QA/SpokenNativQA_azure_asr_GPT4_ZeroShot.py

@@ -0,0 +1,51 @@
+import json
+import re
+
+from llmebench.datasets import SpokenNativQADataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": SpokenNativQADataset,
+        "task": MultiNativQATask,
+        "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa_azure"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+    Please use your expertise to answer the following Arabic question. Answer in Arabic and rate your confidence level from 1 to 10. Provide your response in the following JSON format: {{"answer": "your answer", "score": your confidence score}}. Please provide JSON output only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+    Question: {input_sample['question']}
+    
+    """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["choices"][0]["message"]["content"].strip()
+    content = content.replace("\n", "").strip()
+    if "```json" in content:
+        content = re.search(r"```json(.*)```", content).group(1)
+    return json.loads(content)["answer"]

assets/ar/QA/SpokenNativQA_azure_asr_Jais30bChat_ZeroShot.py

@@ -0,0 +1,48 @@
+import json
+import re
+
+from llmebench.datasets import SpokenNativQADataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "GPT4-o",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": SpokenNativQADataset,
+        "task": MultiNativQATask,
+        "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa_azure"},
+    }
+
+
+def prompt(input_sample):
+    # Define the question prompt
+    question_prompt = f"""
+        Please use your expertise to answer the following Arabic question. Answer in Arabic. Please provide Answer only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+        Question: {input_sample['question']}
+
+        """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are an Arabic AI assistant specialized in providing detailed and accurate answers across various fields. Your task is to deliver clear, concise, and relevant information. 
+    """
+    return [
+        {"role": "user", "content": question_prompt},
+        {"role": "assistant", "content": assistant_prompt},
+    ]
+
+
+def post_process(response):
+    content = response["choices"][0]["message"]["content"].strip()
+    return content