Skip to content

Commit

Permalink
Merge pull request #242 from fabric-testbed/241-cpu-pin-numa-tune
Browse files Browse the repository at this point in the history 
add examples for numa tune and cpu pinning
  • Loading branch information
@kthare10
kthare10 authored Aug 2, 2023
2 parents a1c1a32 + 0e904b6 commit 8f79812
Show file tree
Hide file tree
Showing 4 changed files with 696 additions and 2 deletions.
358 changes: 358 additions & 0 deletions fabric_examples/complex_recipes/iPerf3/iperf3_optimized.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,358 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Create a Slice, Apply performance tuning and Run iPerf3\n",
"\n",
"This notebook creates a slice on a single site with 3 nodes with each node connected to a NIC_Basic and running on a different host. \n",
"All the nodes are connected via local L2 Network. The notebook then depicts an example on how to apply CPU Pinning and Numa Tuning.\n",
"In this notebook, Virtual CPUs for each of the Nodes are pinned to the Physical CPUs residing on the same Numa node as the NIC_Basic connected to the node. In addition, Memory for the Node is also tuned to the same Numa node as the NIC_Basic connected to the node.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Import the FABlib Library\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network\n",
"import ipaddress\n",
"from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n",
"\n",
"fablib = fablib_manager()\n",
"\n",
"fablib.show_config();"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Create the Experiment Slice\n",
"\n",
"The following creates three nodes with basic NICs running on different hosts connected to an isolated local Ethernet. \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# find two available sites\n",
"\n",
"# we will use NIC_Basic to generate traffic so need sites that have >= 3 workers for this example.\n",
"hosts_column_name = 'hosts'\n",
"\n",
"# find two sites with >= 3 hosts\n",
"site1 = fablib.get_random_site(filter_function=lambda x: x[hosts_column_name] > 0)\n",
"print(f\"Site: {site1}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"slice_name = 'iPerf3-tuned-basic'\n",
"\n",
"node1_name='Node1'\n",
"node2_name='Node2'\n",
"\n",
"network_name='net1'\n",
"nic_name = 'nic1'\n",
"model_name = 'NIC_Basic'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Define a subnet to use for the L2 Network\n",
"from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network\n",
"\n",
"subnet = IPv4Network(\"192.168.1.0/24\")\n",
"available_ips = list(subnet)[1:]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Create Slice\n",
"slice = fablib.new_slice(name=slice_name)\n",
"\n",
"net1 = slice.add_l2network(name=network_name, subnet=subnet)\n",
"\n",
"\n",
"hosts = [f\"{site1.lower()}-w1\", f\"{site1.lower()}-w2\", f\"{site1.lower()}-w3\"]\n",
"\n",
"for h in hosts:\n",
"\n",
" # Node1\n",
" node1 = slice.add_node(name=f\"{h}-a\", cores=16, ram=32, site=site1, image='docker_rocky_8', host=f\"{h}.fabric-testbed.net\")\n",
"\n",
" iface1 = node1.add_component(model=model_name, name=nic_name).get_interfaces()[0]\n",
" iface1.set_mode('auto')\n",
" net1.add_interface(iface1)\n",
"\n",
" node1.add_post_boot_upload_directory('node_tools','.')\n",
" node1.add_post_boot_execute('sudo node_tools/host_tune.sh')\n",
" node1.add_post_boot_execute('node_tools/enable_docker.sh {{ _self_.image }} ')\n",
" node1.add_post_boot_execute('docker pull fabrictestbed/slice-vm-rocky8-multitool:0.0.2 ')\n",
"\n",
"#Submit Slice Request\n",
"slice.submit();"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run iPerf3 without any performance tuning\n",
"\n",
"Run iperf3 between every pair of the Nodes before applying performance tuning."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"\n",
"\n",
"source_name = f\"{site1.lower()}-w1-a\"\n",
"source_node = slice.get_node(name=source_name)\n",
"\n",
"source_addr = source_node.get_interface(network_name=network_name).get_ip_addr()\n",
"\n",
"node_names = [f\"{site1.lower()}-w1-a\", f\"{site1.lower()}-w2-a\", f\"{site1.lower()}-w3-a\"]\n",
"\n",
"for i in range(len(node_names)):\n",
" source_node = slice.get_node(name=node_names[i])\n",
" source_addr = source_node.get_interface(network_name=network_name).get_ip_addr()\n",
" \n",
" for j in range(len(node_names)):\n",
" if i==j:\n",
" continue\n",
" print(\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\")\n",
" stdout1, stderr1 = source_node.execute(\"docker run -d --rm \"\n",
" \"--network host \"\n",
" \"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 \"\n",
" \"iperf3 -s -1\"\n",
" , quiet=True, output_file=f\"{source_node.get_name()}-no-tune.log\");\n",
"\n",
" print(f\"Source: {node_names[i]} to Dest: {node_names[j]}\")\n",
" dest_node = slice.get_node(name=node_names[j])\n",
"\n",
" stdout2, stderr2 = dest_node.execute(\"docker run --rm \"\n",
" \"--network host \"\n",
" \"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 \"\n",
" f\"iperf3 -c {source_addr} -P 10 -t 30 -i 10 -O 10\"\n",
" , quiet=False, output_file=f\"{dest_node.get_name()}-no-tune.log\");\n",
" print(\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Perform CPU Pinning and Numa Tuning"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"\n",
"for node in slice.get_nodes():\n",
" # Pin all vCPUs for VM to same Numa node as the component\n",
" node.pin_cpu(component_name=nic_name)\n",
" \n",
" # User can also pass in the range of the vCPUs to be pinned \n",
" #node.pin_cpu(component_name=nic_name, cpu_range_to_pin=\"0-3\")\n",
" \n",
" # Pin memmory for VM to same Numa node as the components\n",
" node.numa_tune()\n",
" \n",
" # Reboot the VM\n",
" node.os_reboot()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Re-configure Network Interfaces and Enable Docker post reboot"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"\n",
"# Wait for the SSH Connectivity to be back\n",
"slice.wait_ssh()\n",
"\n",
"# Re-configuring the Network\n",
"for node in slice.get_nodes():\n",
" node.config()\n",
" stdout1, stderr1 = node.execute(\"sudo systemctl enable docker\")\n",
" stdout1, stderr1 = node.execute(\"sudo systemctl start docker\") \n",
" stdout1, stderr1 = node.execute(\"sudo docker pull fabrictestbed/slice-vm-rocky8-multitool:0.0.2 \")\n",
" stdout1, stderr1 = node.execute(\"sudo systemctl enable docker\")\n",
" stdout1, stderr1 = node.execute(\"sudo systemctl start docker\") "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Verify IP Addresses are configured"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"slice.list_interfaces();"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run iperf3 after performance tuning\n",
"\n",
"Run iperf3 between every pair of the Nodes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"\n",
"\n",
"source_name = f\"{site1.lower()}-w1-a\"\n",
"source_node = slice.get_node(name=source_name)\n",
"\n",
"source_addr = source_node.get_interface(network_name=network_name).get_ip_addr()\n",
"\n",
"node_names = [f\"{site1.lower()}-w1-a\", f\"{site1.lower()}-w2-a\", f\"{site1.lower()}-w3-a\"]\n",
"\n",
"for i in range(len(node_names)):\n",
" source_node = slice.get_node(name=node_names[i])\n",
" source_addr = source_node.get_interface(network_name=network_name).get_ip_addr()\n",
" \n",
" for j in range(len(node_names)):\n",
" if i==j:\n",
" continue\n",
" print(\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\")\n",
" stdout1, stderr1 = source_node.execute(\"docker run -d --rm \"\n",
" \"--network host \"\n",
" \"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 \"\n",
" \"iperf3 -s -1\"\n",
" , quiet=True, output_file=f\"{source_node.get_name()}.log\");\n",
"\n",
" print(f\"Source: {node_names[i]} to Dest: {node_names[j]}\")\n",
" dest_node = slice.get_node(name=node_names[j])\n",
"\n",
" stdout2, stderr2 = dest_node.execute(\"docker run --rm \"\n",
" \"--network host \"\n",
" \"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 \"\n",
" f\"iperf3 -c {source_addr} -P 10 -t 30 -i 10 -O 10\"\n",
" , quiet=False, output_file=f\"{dest_node.get_name()}.log\");\n",
" print(\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Delete the Slice\n",
"\n",
"Please delete your slice when you are done with your experiment."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"slice = fablib.get_slice(slice_name)\n",
"slice.delete()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 8f79812

Please sign in to comment.