Skip to content

Commit 1d2e3b9

Browse files
cavinkavicavinkavi
committed
Integrate Azure Functions deployment
Signed-off-by: Kavithran <104263022+cavinkavi@users.noreply.github.com>
1 parent eaa1d2d commit 1d2e3b9

19 files changed

+946
-32
lines changed

‎.github/workflows/e2e_azure.yaml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: End-to-End Azure Functions Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main # Trigger the workflow when code is pushed to the main branch
7+
pull_request:
8+
branches:
9+
- main # Trigger the workflow when the PR targets the main branch
10+
workflow_dispatch: # Allows manual triggering of the workflow
11+
12+
env:
13+
GOOS: linux
14+
GO111MODULE: on
15+
16+
jobs:
17+
test-azure:
18+
name: Test E2E Azure Functions Cloud Deployment
19+
runs-on: ubuntu-20.04
20+
env:
21+
AZURE_APP_ID: ${{ secrets.AZURE_APP_ID }}
22+
AZURE_PASSWORD: ${{ secrets.AZURE_PASSWORD }}
23+
AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
24+
25+
steps:
26+
- name: Check if environment variables are set # Validate secrets are passed
27+
run: |
28+
if [[ -z "$AZURE_APP_ID" ]]; then
29+
echo "AZURE_APP_ID is not set. Please check if secrets.AZURE_APP_ID is in the repository."
30+
exit 1
31+
fi
32+
if [[ -z "$AZURE_PASSWORD" ]]; then
33+
echo "AZURE_PASSWORD is not set. Please check if secrets.AZURE_PASSWORD is in the repository."
34+
exit 1
35+
fi
36+
if [[ -z "$AZURE_TENANT" ]]; then
37+
echo "AZURE_TENANT is not set. Please check if secrets.AZURE_TENANT is in the repository."
38+
exit 1
39+
fi
40+
41+
- name: Checkout GitHub Repository
42+
uses: actions/checkout@v4
43+
with:
44+
lfs: true
45+
46+
- name: Install Azure CLI
47+
run: |
48+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
49+
az --version
50+
51+
- name: Install Golang
52+
uses: actions/setup-go@v5
53+
with:
54+
go-version: 1.22
55+
56+
- name: Set up Python 3.10
57+
uses: actions/setup-python@v4
58+
with:
59+
python-version: '3.10'
60+
61+
- name: Azure CLI Login Using Service Principal
62+
run: az login --service-principal --username $AZURE_APP_ID --password $AZURE_PASSWORD --tenant $AZURE_TENANT
63+
64+
- name: Build and Run Loader
65+
run: go run cmd/loader.go --config cmd/config_azure_trace.json
66+
67+
- name: Check the output
68+
run: test -f "data/out/experiment_duration_5.csv" && test $(grep true data/out/experiment_duration_5.csv | wc -l) -eq 0 # test the output file for errors (true means failure to invoke)

‎.github/workflows/unit-tests-azure.yaml

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Unit Tests for Azure Functions
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
workflow_dispatch:
11+
12+
jobs:
13+
test:
14+
name: Run Azure Functions Unit Tests
15+
runs-on: ubuntu-20.04
16+
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
21+
- name: Set up Go
22+
uses: actions/setup-go@v5
23+
with:
24+
go-version: '1.22'
25+
26+
- name: Run Workload Copy Test
27+
run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestCopyPythonWorkload
28+
29+
- name: Run Zip File Health Test
30+
run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestZipHealth
31+
32+
- name: Run Deploy Function Test
33+
run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestDeployFunction
34+
35+
- name: Run Cleanup Test
36+
run: go test -v ./pkg/driver/deployment/azure_functions_test.go -run TestCleanup

‎azurefunctions_setup/azurefunctionsconfig.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# azurefunctionsconfig.yaml
2+
azurefunctionsconfig:
3+
resource_group: ExperimentResourceGroup # Name of the resource group
4+
storage_account_name: testinvitrostorage # Name of the storage account
5+
function_app_name: testinvitrofunctionapp # Name of the function app
6+
location: EastUS # Region where resource created

‎azurefunctions_setup/host.json

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"version": "2.0",
3+
"logging": {
4+
"applicationInsights": {
5+
"samplingSettings": {
6+
"isEnabled": true,
7+
"excludedTypes": "Request"
8+
}
9+
}
10+
},
11+
"extensionBundle": {
12+
"id": "Microsoft.Azure.Functions.ExtensionBundle",
13+
"version": "[4.*, 5.0.0)"
14+
}
15+
16+
}

‎azurefunctions_setup/local.settings.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"IsEncrypted": false,
3+
"Values": {
4+
"FUNCTIONS_WORKER_RUNTIME": "python",
5+
"AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
6+
"AzureWebJobsStorage": ""
7+
}
8+
9+
}

‎azurefunctions_setup/requirements.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
azure-functions
2+
numpy>=1.21,<1.26
3+
psutil>=5.9,<6.0

‎azurefunctions_setup/shared_azure_workload/azurefunctionsworkload.py

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import time
2+
import socket
3+
import json
4+
import azure.functions as func
5+
import logging
6+
7+
from .exec_func import execute_function
8+
9+
# Global variable for hostname
10+
hostname = socket.gethostname()
11+
12+
def main(req: func.HttpRequest) -> func.HttpResponse:
13+
logging.info("Processing request.")
14+
15+
start_time = time.time()
16+
17+
# Parse JSON request body
18+
try:
19+
req_body = req.get_json()
20+
logging.info(f"Request body: {req_body}")
21+
except ValueError:
22+
logging.error("Invalid JSON received.")
23+
return func.HttpResponse(
24+
json.dumps({"error": "Invalid JSON"}),
25+
status_code=400,
26+
mimetype="application/json"
27+
)
28+
29+
runtime_milliseconds = req_body.get('RuntimeInMilliSec', 1000)
30+
memory_mebibytes = req_body.get('MemoryInMebiBytes', 128)
31+
32+
logging.info(f"Runtime requested: {runtime_milliseconds} ms, Memory: {memory_mebibytes} MiB")
33+
34+
# Directly call the execute_function
35+
duration = execute_function("",runtime_milliseconds,memory_mebibytes)
36+
result_msg = f"Workload completed in {duration} microseconds"
37+
38+
# Prepare the response
39+
response = {
40+
"Status": "Success",
41+
"Function": req.url.split("/")[-1],
42+
"MachineName": hostname,
43+
"ExecutionTime": int((time.time() - start_time) * 1_000_000), # Total time (includes HTTP, workload, and response prep)
44+
"DurationInMicroSec": duration, # Time spent on the workload itself
45+
"MemoryUsageInKb": memory_mebibytes * 1024,
46+
"Message": result_msg
47+
}
48+
49+
logging.info(f"Response: {response}")
50+
51+
return func.HttpResponse(
52+
json.dumps(response),
53+
status_code=200,
54+
mimetype="application/json"
55+
)

‎azurefunctions_setup/shared_azure_workload/exec_func.py

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import math
2+
from time import process_time_ns
3+
from numpy import empty, float32
4+
from psutil import virtual_memory
5+
6+
7+
def execute_function(input, runTime, totalMem):
8+
startTime = process_time_ns()
9+
10+
chunkSize = 2**10 # size of a kb or 1024
11+
totalMem = totalMem*(2**10) # convert Mb to kb
12+
memory = virtual_memory()
13+
used = (memory.total - memory.available) // chunkSize # convert to kb
14+
additional = max(1, (totalMem - used))
15+
array = empty(additional*chunkSize, dtype=float32) # make an uninitialized array of that size, uninitialized to keep it fast
16+
# convert to ns
17+
runTime = (runTime - 1)*(10**6) # -1 because it should be slighly below that runtime
18+
memoryIndex = 0
19+
while process_time_ns() - startTime < runTime:
20+
for i in range(0, chunkSize):
21+
sin_i = math.sin(i)
22+
cos_i = math.cos(i)
23+
sqrt_i = math.sqrt(i)
24+
array[memoryIndex + i] = sin_i
25+
memoryIndex = (memoryIndex + chunkSize) % additional*chunkSize
26+
return (process_time_ns() - startTime) // 1000

‎azurefunctions_setup/shared_azure_workload/function.json

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"bindings": [
3+
{
4+
"authLevel": "anonymous",
5+
"type": "httpTrigger",
6+
"direction": "in",
7+
"name": "req",
8+
"methods": ["post"]
9+
},
10+
{
11+
"type": "http",
12+
"direction": "out",
13+
"name": "$return"
14+
}
15+
],
16+
"scriptFile": "azurefunctionsworkload.py"
17+
}
18+

‎cmd/config_azure_trace.json

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"Seed": 42,
3+
4+
"Platform": "AzureFunctions",
5+
"InvokeProtocol" : "http1",
6+
"EndpointPort": 80,
7+
8+
"BusyLoopOnSandboxStartup": false,
9+
10+
"TracePath": "data/traces/example",
11+
"Granularity": "minute",
12+
"OutputPathPrefix": "data/out/experiment",
13+
"IATDistribution": "exponential",
14+
"CPULimit": "1vCPU",
15+
"ExperimentDuration": 5,
16+
"WarmupDuration": 0,
17+
18+
"IsPartiallyPanic": false,
19+
"EnableZipkinTracing": false,
20+
"EnableMetricsScrapping": false,
21+
"MetricScrapingPeriodSeconds": 15,
22+
"AutoscalingMetric": "concurrency",
23+
24+
"GRPCConnectionTimeoutSeconds": 15,
25+
"GRPCFunctionTimeoutSeconds": 900,
26+
27+
"DAGMode": false
28+
}

‎cmd/loader.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ func main() {
9696
"AWSLambda",
9797
"Dirigent",
9898
"Dirigent-Dandelion",
99+
"AzureFunctions",
99100
}
100101

101102
if !slices.Contains(supportedPlatforms, cfg.Platform) {
@@ -151,7 +152,7 @@ func parseYAMLSpecification(cfg *config.LoaderConfiguration) string {
151152
case "firecracker":
152153
return "workloads/firecracker/trace_func_go.yaml"
153154
default:
154-
if cfg.Platform != "Dirigent" && cfg.Platform != "Dirigent-Dandelion" {
155+
if cfg.Platform != "Dirigent" && cfg.Platform != "Dirigent-Dandelion" && cfg.Platform != "AzureFunctions" {
155156
log.Fatal("Invalid 'YAMLSelector' parameter.")
156157
}
157158
}

‎docs/loader.md

+45-1
Original file line numberDiff line numberDiff line change
@@ -262,4 +262,48 @@ Note:
262262
- Under `Manage Quota`, select `AWS Lambda` service and click `View quotas` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas))
263263
- Under `Quota name`, select `Concurrent executions` and click `Request increase at account level` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas/L-B99A9384))
264264
- Under `Increase quota value`, input `1000` and click `Request`
265-
- Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
265+
- Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
266+
267+
## Using Azure Functions
268+
269+
**Pre-requisites:**
270+
1. Microsoft Azure account with an active subscription ID
271+
2. Existing Service Principal for authentication (refer to Notes section)
272+
3. Go installed
273+
4. Python3 installed
274+
275+
**Quick Setup for Azure Deployment:**
276+
1. Install the Azure CLI and verify installation:
277+
```bash
278+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
279+
az --version
280+
```
281+
2. Use existing Service Principal credentials in order login to Azure.
282+
```bash
283+
az login --service-principal --username $AZURE_APP_ID --password $AZURE_PASSWORD --tenant $AZURE_TENANT
284+
```
285+
> Refer to Note section for generation of Service Principal credentials
286+
3. Start the Azure Functions deployment experiment:
287+
```bash
288+
go run cmd/loader.go --config cmd/config_azure_trace.json
289+
```
290+
---
291+
Notes:
292+
293+
- Service Principal must be created before running experiment, as some environments do not have browsers (e.g. CloudLab). Perform these steps in an environment that allows launching of browser and use the generated credentials.
294+
- Log in as a user (Note: This will open a browser window to select Azure account):
295+
```bash
296+
az login
297+
```
298+
- Create an Azure Service Principal:
299+
```bash
300+
az ad sp create-for-rbac --name "InVitro" --role Contributor --scopes /subscriptions/<your-subscription-id>
301+
```
302+
- Set the following values in the environment that the experiment is being run and return to Step 2 of setup:
303+
```bash
304+
export AZURE_APP_ID=<appId>
305+
export AZURE_PASSWORD=<password>
306+
export AZURE_TENANT=<tenant>
307+
```
308+
- Current deployment is via ZIP
309+
- Python is used for deployment workload as Go is not supported in Consumption Plan

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ require (
1717

1818
require (
1919
github.com/aws/aws-lambda-go v1.47.0
20-
github.com/stretchr/testify v1.10.0
2120
github.com/containerd/log v0.1.0
2221
github.com/google/uuid v1.6.0
22+
github.com/stretchr/testify v1.10.0
2323
github.com/vhive-serverless/vSwarm/utils/protobuf/helloworld v0.0.0-20240827121957-11be651eb39a
2424
github.com/vhive-serverless/vSwarm/utils/tracing/go v0.0.0-20240827121957-11be651eb39a
2525
go.mongodb.org/mongo-driver v1.17.1

‎pkg/common/utilities.go

+25-1
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@ package common
2626

2727
import (
2828
"hash/fnv"
29+
"io"
2930
"log"
3031
"math/rand"
32+
"os"
3133
"strconv"
3234
"strings"
3335
)
@@ -146,4 +148,26 @@ func GetName(function *Function) int {
146148
log.Fatal(err)
147149
}
148150
return functionId
149-
}
151+
}
152+
153+
// Helper function to copy files
154+
func CopyFile(src, dst string) error {
155+
sourceFile, err := os.Open(src)
156+
if err != nil {
157+
return err
158+
}
159+
defer sourceFile.Close()
160+
161+
destFile, err := os.Create(dst)
162+
if err != nil {
163+
return err
164+
}
165+
defer destFile.Close()
166+
167+
_, err = io.Copy(destFile, sourceFile)
168+
if err != nil {
169+
return err
170+
}
171+
172+
return destFile.Sync()
173+
}

0 commit comments

Comments
 (0)