Skip to content

Commit

Permalink
Merge pull request #177 from fabric-testbed/additiona-acceptance-tests
Browse files Browse the repository at this point in the history 
Updated previous acceptance tests and added all possible tests
  • Loading branch information
@kthare10
kthare10 authored Oct 8, 2022
2 parents 86eb2ca + ca1a2b0 commit 4a358e9
Show file tree
Hide file tree
Showing 11 changed files with 3,901 additions and 344 deletions.
188 changes: 19 additions & 169 deletions fabric_examples/acceptance_testing/Acceptance Tests 3.1.2/functional_test_3.1.2-gpu.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,31 +24,9 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------------------------------- --------------------------------------------------\n",
"credmgr_host cm.fabric-testbed.net\n",
"orchestrator_host orchestrator.fabric-testbed.net\n",
"fabric_token /home/fabric/.tokens.json\n",
"project_id 990d8a8b-7e50-4d13-a3be-0f133ffa8653\n",
"bastion_username ibaldin_0000241998\n",
"bastion_key_filename /home/fabric/work/fabric_config/fabric_bastion_key\n",
"bastion_public_addr bastion-1.fabric-testbed.net\n",
"bastion_passphrase None\n",
"slice_public_key_file /home/fabric/work/fabric_config/slice_key.pub\n",
"slice_private_key_file /home/fabric/work/fabric_config/slice_key\n",
"fabric_slice_private_key_passphrase None\n",
"fablib_log_file /tmp/fablib/fablib.log\n",
"fablib_log_level INFO\n",
"----------------------------------- --------------------------------------------------\n"
]
}
],
"outputs": [],
"source": [
"from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n",
"\n",
Expand All @@ -68,7 +46,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -94,7 +72,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -115,32 +93,9 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"----------- ---------------------------------------------------------------------------\n",
"Slice Name Slice Test TACC on tacc-w1.fabric-testbed.net on 2022-08-31 01:09:04.792628\n",
"Slice ID efec5ff9-0099-4f72-9009-a1ab1f55c44e\n",
"Slice State StableOK\n",
"Lease End 2022-09-01 01:09:13 +0000\n",
"----------- ---------------------------------------------------------------------------\n",
"\n",
"Retry: 13, Time: 142 sec\n",
"\n",
"ID Name Site Host Cores RAM Disk Image Management IP State Error\n",
"------------------------------------ ------ ------ -------------------------- ------- ----- ------ --------------- --------------- ------- -------\n",
"c70530c8-4bf6-415c-8f4a-9a0dbf2f183c Node1 TACC tacc-w1.fabric-testbed.net 10 32 100 default_rocky_8 129.114.110.76 Active\n",
"\n",
"Time to stable 142 seconds\n",
"Running post_boot_config ... Time to post boot config 143 seconds\n"
]
}
],
"outputs": [],
"source": [
"try:\n",
" #Create Slice\n",
Expand Down Expand Up @@ -172,22 +127,9 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------- ---------------------------------------------------------------------------\n",
"Slice Name Slice Test TACC on tacc-w1.fabric-testbed.net on 2022-08-31 01:09:04.792628\n",
"Slice ID efec5ff9-0099-4f72-9009-a1ab1f55c44e\n",
"Slice State StableOK\n",
"Lease End 2022-09-01 01:09:13 +0000\n",
"----------- ---------------------------------------------------------------------------\n"
]
}
],
"outputs": [],
"source": [
"try:\n",
" slice = fablib.get_slice(name=slice_name)\n",
Expand All @@ -207,40 +149,9 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------------- ------------------------------------------------------------------------------------------------------------------------\n",
"ID c70530c8-4bf6-415c-8f4a-9a0dbf2f183c\n",
"Name Node1\n",
"Cores 10\n",
"RAM 32\n",
"Disk 100\n",
"Image default_rocky_8\n",
"Image Type qcow2\n",
"Host tacc-w1.fabric-testbed.net\n",
"Site TACC\n",
"Management IP 129.114.110.76\n",
"Reservation State Active\n",
"Error Message\n",
"SSH Command ssh -i /home/fabric/work/fabric_config/slice_key -J ibaldin_0000241998@bastion-1.fabric-testbed.net rocky@129.114.110.76\n",
"----------------- ------------------------------------------------------------------------------------------------------------------------\n",
"----------- ----------------------------------------------------------\n",
"Name Node1-gpu1\n",
"Details NVIDIA Corporation TU102GL [Quadro RTX 6000/8000] (rev a1)\n",
"Disk (G) 0\n",
"Units 1\n",
"PCI Address 0000:25:00.0\n",
"Model GPU_RTX6000\n",
"Type GPU\n",
"----------- ----------------------------------------------------------\n"
]
}
],
"outputs": [],
"source": [
"try:\n",
" node = slice.get_node(name) \n",
Expand All @@ -266,22 +177,9 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"stdout: \n",
"Installed:\n",
" pciutils-3.7.0-1.el8.x86_64 \n",
"\n",
"00:07.0 3D controller: NVIDIA Corporation TU102GL [Quadro RTX 6000/8000] (rev a1)\n",
"\n"
]
}
],
"outputs": [],
"source": [
"command = \"sudo dnf install -q -y pciutils && lspci | grep 'NVIDIA\\|3D controller'\"\n",
"try:\n",
Expand All @@ -302,18 +200,9 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Installing CUDA...\n",
"Done installing CUDA. Now, reboot for the changes to take effect.\n"
]
}
],
"outputs": [],
"source": [
"commands = [\n",
" 'sudo dnf install -q -y epel-release',\n",
Expand All @@ -338,20 +227,9 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"sudo reboot\n",
"Waiting for slice . Slice state: StableOK\n",
"Waiting for ssh in slice ... ssh successful\n",
"Now testing SSH abilites to reconnect...Reconnected!\n"
]
}
],
"outputs": [],
"source": [
"reboot = 'sudo reboot'\n",
"try:\n",
Expand Down Expand Up @@ -380,37 +258,9 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"stdout: Wed Aug 31 01:23:03 2022 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 515.65.01 Driver Version: 515.65.01 CUDA Version: 11.7 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"| | | MIG M. |\n",
"|===============================+======================+======================|\n",
"| 0 Quadro RTX 6000 Off | 00000000:00:07.0 Off | 0 |\n",
"| N/A 30C P0 54W / 250W | 0MiB / 23040MiB | 6% Default |\n",
"| | | N/A |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: |\n",
"| GPU GI CI PID Type Process name GPU Memory |\n",
"| ID ID Usage |\n",
"|=============================================================================|\n",
"| No running processes found |\n",
"+-----------------------------------------------------------------------------+\n",
"\n"
]
}
],
"outputs": [],
"source": [
"try:\n",
" stdout, stderr = node.execute(\"nvidia-smi\")\n",
Expand All @@ -428,7 +278,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand Down
Loading

0 comments on commit 4a358e9

Please sign in to comment.