Integrate Azure Functions deployment with In-Vitro

.github/workflows/e2e_azure.yaml

@@ -0,0 +1,68 @@
+name: End-to-End Azure Functions Tests
+
+on:
+  push:
+    branches:
+      - main # Trigger the workflow when code is pushed to the main branch
+  pull_request:
+    branches:
+      - main # Trigger the workflow when the PR targets the main branch
+  workflow_dispatch: # Allows manual triggering of the workflow
+
+env:
+  GOOS: linux
+  GO111MODULE: on
+
+jobs:
+  test-azure:
+    name: Test E2E Azure Functions Cloud Deployment
+    runs-on: ubuntu-20.04
+    env:
+      AZURE_APP_ID: ${{ secrets.AZURE_APP_ID }}
+      AZURE_PASSWORD: ${{ secrets.AZURE_PASSWORD }}
+      AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
+
+    steps:
+      - name: Check if environment variables are set  # Validate secrets are passed
+        run: |
+          if [[ -z "$AZURE_APP_ID" ]]; then
+            echo "AZURE_APP_ID is not set. Please check if secrets.AZURE_APP_ID is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_PASSWORD" ]]; then
+            echo "AZURE_PASSWORD is not set. Please check if secrets.AZURE_PASSWORD is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_TENANT" ]]; then
+            echo "AZURE_TENANT is not set. Please check if secrets.AZURE_TENANT is in the repository."
+            exit 1
+          fi
+
+      - name: Checkout GitHub Repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+
+      - name: Install Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Azure CLI Login Using Service Principal
+        run: az login --service-principal --username $AZURE_APP_ID --password $AZURE_PASSWORD --tenant $AZURE_TENANT
+
+      - name: Build and Run Loader
+        run: go run cmd/loader.go --config cmd/config_azure_trace.json
+
+      - name: Check the output
+        run: test -f "data/out/experiment_duration_5.csv" && test $(grep true data/out/experiment_duration_5.csv | wc -l) -eq 0 # test the output file for errors (true means failure to invoke)

.github/workflows/unit-tests-azure.yaml

@@ -0,0 +1,36 @@
+name: Unit Tests for Azure Functions
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Run Azure Functions Unit Tests
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Run Workload Copy Test
+        run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestCopyPythonWorkload
+
+      - name: Run Zip File Health Test
+        run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestZipHealth
+
+      - name: Run Deploy Function Test
+        run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestDeployFunction
+
+      - name: Run Cleanup Test
+        run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestCleanup

azurefunctions_setup/azurefunctionsconfig.yaml

@@ -0,0 +1,6 @@
+# azurefunctionsconfig.yaml
+azurefunctionsconfig:
+  resource_group: ExperimentResourceGroup  # Name of the resource group
+  storage_account_name: testinvitrostorage  # Name of the storage account
+  function_app_name: testinvitrofunctionapp  # Name of the function app
+  location: EastUS  # Region where resource created

azurefunctions_setup/host.json

@@ -0,0 +1,16 @@
+{
+    "version": "2.0",
+    "logging": {
+      "applicationInsights": {
+        "samplingSettings": {
+          "isEnabled": true,
+          "excludedTypes": "Request"
+        }
+      }
+    },
+    "extensionBundle": {
+      "id": "Microsoft.Azure.Functions.ExtensionBundle",
+      "version": "[4.*, 5.0.0)"
+    }
+
+}

azurefunctions_setup/local.settings.json

@@ -0,0 +1,9 @@
+{
+    "IsEncrypted": false,
+    "Values": {
+      "FUNCTIONS_WORKER_RUNTIME": "python",
+      "AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
+      "AzureWebJobsStorage": ""
+    }
+
+}

azurefunctions_setup/requirements.txt

@@ -0,0 +1,3 @@
+azure-functions
+numpy>=1.21,<1.26
+psutil>=5.9,<6.0

azurefunctions_setup/shared_azure_workload/azurefunctionsworkload.py

@@ -0,0 +1,55 @@
+import time
+import socket
+import json
+import azure.functions as func
+import logging
+
+from .exec_func import execute_function
+
+# Global variable for hostname
+hostname = socket.gethostname()
+
+def main(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info("Processing request.")
+
+    start_time = time.time()
+
+    # Parse JSON request body
+    try:
+        req_body = req.get_json()
+        logging.info(f"Request body: {req_body}")
+    except ValueError:
+        logging.error("Invalid JSON received.")
+        return func.HttpResponse(
+            json.dumps({"error": "Invalid JSON"}),
+            status_code=400,
+            mimetype="application/json"
+        )
+
+    runtime_milliseconds = req_body.get('RuntimeInMilliSec', 1000)
+    memory_mebibytes = req_body.get('MemoryInMebiBytes', 128)
+
+    logging.info(f"Runtime requested: {runtime_milliseconds} ms, Memory: {memory_mebibytes} MiB")
+
+    # Directly call the execute_function
+    duration = execute_function("",runtime_milliseconds,memory_mebibytes)
+    result_msg = f"Workload completed in {duration} microseconds"
+
+    # Prepare the response
+    response = {
+        "Status": "Success",
+        "Function": req.url.split("/")[-1],
+        "MachineName": hostname,
+        "ExecutionTime": int((time.time() - start_time) * 1_000_000),  # Total time (includes HTTP, workload, and response prep)
+        "DurationInMicroSec": duration,  # Time spent on the workload itself
+        "MemoryUsageInKb": memory_mebibytes * 1024,
+        "Message": result_msg
+    }
+
+    logging.info(f"Response: {response}")
+
+    return func.HttpResponse(
+        json.dumps(response),
+        status_code=200,
+        mimetype="application/json"
+    )

azurefunctions_setup/shared_azure_workload/exec_func.py

@@ -0,0 +1,26 @@
+import math
+from time import process_time_ns
+from numpy import empty, float32
+from psutil import virtual_memory
+
+
+def execute_function(input, runTime, totalMem):
+    startTime = process_time_ns()
+
+    chunkSize = 2**10 # size of a kb or 1024
+    totalMem = totalMem*(2**10) # convert Mb to kb
+    memory = virtual_memory()
+    used = (memory.total - memory.available) // chunkSize # convert to kb
+    additional = max(1, (totalMem - used))
+    array = empty(additional*chunkSize, dtype=float32) # make an uninitialized array of that size, uninitialized to keep it fast
+    # convert to ns
+    runTime = (runTime - 1)*(10**6) # -1 because it should be slighly below that runtime
+    memoryIndex = 0
+    while process_time_ns() - startTime < runTime:
+        for i in range(0, chunkSize):
+            sin_i = math.sin(i)
+            cos_i = math.cos(i)
+            sqrt_i = math.sqrt(i)
+            array[memoryIndex + i] = sin_i
+        memoryIndex = (memoryIndex + chunkSize) % additional*chunkSize
+    return (process_time_ns() - startTime) // 1000

azurefunctions_setup/shared_azure_workload/function.json

@@ -0,0 +1,18 @@
+{
+    "bindings": [
+      {
+        "authLevel": "anonymous",
+        "type": "httpTrigger",
+        "direction": "in",
+        "name": "req",
+        "methods": ["post"]
+      },
+      {
+        "type": "http",
+        "direction": "out",
+        "name": "$return"
+      }
+    ],
+    "scriptFile": "azurefunctionsworkload.py"
+}
+

cmd/config_azure_trace.json

@@ -0,0 +1,28 @@
+{
+    "Seed": 42,
+
+    "Platform": "AzureFunctions",
+    "InvokeProtocol" : "http1", 
+    "EndpointPort": 80, 
+
+    "BusyLoopOnSandboxStartup": false,
+
+    "TracePath": "data/traces/example",
+    "Granularity": "minute",
+    "OutputPathPrefix": "data/out/experiment",
+    "IATDistribution": "exponential",
+    "CPULimit": "1vCPU",    
+    "ExperimentDuration": 5,
+    "WarmupDuration": 0,
+
+    "IsPartiallyPanic": false,
+    "EnableZipkinTracing": false,
+    "EnableMetricsScrapping": false,
+    "MetricScrapingPeriodSeconds": 15,
+    "AutoscalingMetric": "concurrency",
+
+    "GRPCConnectionTimeoutSeconds": 15,     
+    "GRPCFunctionTimeoutSeconds": 900,     
+
+    "DAGMode": false
+}

cmd/loader.go

@@ -96,6 +96,7 @@ func main() {
 		"AWSLambda",
 		"Dirigent",
 		"Dirigent-Dandelion",
+		"AzureFunctions",
 	}
 
 	if !slices.Contains(supportedPlatforms, cfg.Platform) {
@@ -151,7 +152,7 @@ func parseYAMLSpecification(cfg *config.LoaderConfiguration) string {
 	case "firecracker":
 		return "workloads/firecracker/trace_func_go.yaml"
 	default:
-		if cfg.Platform != "Dirigent" && cfg.Platform != "Dirigent-Dandelion" {
+		if cfg.Platform != "Dirigent" && cfg.Platform != "Dirigent-Dandelion" && cfg.Platform != "AzureFunctions" {
 			log.Fatal("Invalid 'YAMLSelector' parameter.")
 		}
 	}

docs/loader.md

@@ -262,4 +262,48 @@ Note:
   - Under `Manage Quota`, select `AWS Lambda` service and click `View quotas` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas))
   - Under `Quota name`, select `Concurrent executions` and click `Request increase at account level` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas/L-B99A9384))
   - Under `Increase quota value`, input `1000` and click `Request`
-  - Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
+  - Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
+
+## Using Azure Functions
+
+**Pre-requisites:**
+1. Microsoft Azure account with an active subscription ID
+2. Existing Service Principal for authentication (refer to Notes section)
+3. Go installed
+4. Python3 installed
+
+**Quick Setup for Azure Deployment:**
+1. Install the Azure CLI and verify installation:
+    ```bash
+    curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+    az --version
+    ```
+2. Use existing Service Principal credentials in order login to Azure.
+    ```bash
+    az login --service-principal --username $AZURE_APP_ID --password $AZURE_PASSWORD --tenant $AZURE_TENANT
+    ```
+   > Refer to Note section for generation of Service Principal credentials
+3. Start the Azure Functions deployment experiment:
+    ```bash
+    go run cmd/loader.go --config cmd/config_azure_trace.json
+    ```
+---
+Notes:
+
+- Service Principal must be created before running experiment, as some environments do not have browsers (e.g. CloudLab). Perform these steps in an environment that allows launching of browser and use the generated credentials. 
+  - Log in as a user (Note: This will open a browser window to select Azure account):
+     ```bash
+     az login
+     ```
+  - Create an Azure Service Principal: 
+     ```bash
+     az ad sp create-for-rbac --name "InVitro" --role Contributor --scopes /subscriptions/<your-subscription-id>
+     ```
+  - Set the following values in the environment that the experiment is being run and return to Step 2 of setup: 
+     ```bash
+     export AZURE_APP_ID=<appId>
+     export AZURE_PASSWORD=<password>
+     export AZURE_TENANT=<tenant>
+     ```
+- Current deployment is via ZIP
+- Python is used for deployment workload as Go is not supported in Consumption Plan

pkg/common/utilities.go

@@ -27,8 +27,10 @@ package common
 import (
 	"encoding/json"
 	"hash/fnv"
+	"io"
 	"log"
 	"math/rand"
+	"os"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -152,6 +154,28 @@ func GetName(function *Function) int {
 	return functionId
 }
 
+// Helper function to copy files
+func CopyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return err
+	}
+
+	return destFile.Sync()
+}
+
 func DeepCopy[T any](a T) (T, error) {
 	var b T
 	byt, err := json.Marshal(a)