Skip to content

Commit

Permalink
Initial version of bulk update of config store when volttron is not r…
Browse files Browse the repository at this point in the history 
…unning

issue VOLTTRON#3121
  • Loading branch information
@schandrika
schandrika committed Sep 26, 2023
1 parent d2f3877 commit 18b03e9
Showing 1 changed file with 182 additions and 37 deletions.
219 changes: 182 additions & 37 deletions volttron/platform/instance_setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
# }}}
import argparse
import hashlib
import json
import os
import sys
import tempfile
Expand All @@ -57,10 +58,13 @@
from volttron.platform import jsonapi
from volttron.platform.agent.known_identities import PLATFORM_WEB, PLATFORM_DRIVER, VOLTTRON_CENTRAL
from volttron.platform.agent.utils import get_platform_instance_name, wait_for_volttron_startup, \
is_volttron_running, wait_for_volttron_shutdown, setup_logging
is_volttron_running, wait_for_volttron_shutdown, setup_logging, format_timestamp, get_aware_utc_now, \
parse_json_config
from volttron.utils import get_hostname
from volttron.utils.prompt import prompt_response, y, n, y_or_n
from . import get_home, get_services_core, set_home
from volttron.platform.agent.utils import load_config as load_yml_or_json
from volttron.platform.store import process_raw_config

if is_rabbitmq_available():
from bootstrap import install_rabbit, default_rmq_dir
Expand Down Expand Up @@ -155,32 +159,32 @@ def _is_bound_already(address):
return already_bound


def fail_if_instance_running(args):
def fail_if_instance_running(message, prompt=True):

home = get_home()

if os.path.exists(home) and\
is_volttron_running(home):
global use_active
use_active = prompt_response(
"The VOLTTRON Instance is currently running. "
"Installing agents to an active instance may overwrite currently installed "
"and active agents on the platform, causing undesirable behavior. "
"Would you like to continue?",
valid_answers=y_or_n,
default='Y')
if use_active in y:
return
if prompt:
global use_active
use_active = prompt_response(
message +
"Would you like to continue?",
valid_answers=y_or_n,
default='Y')
if use_active in y:
return
else:
print("""
print(message)
print("""
Please execute:
volttron-ctl shutdown --platform
to stop the instance.
""")

sys.exit()
sys.exit(1)


def fail_if_not_in_src_root():
Expand All @@ -189,7 +193,7 @@ def fail_if_not_in_src_root():
print("""
volttron-cfg needs to be run from the volttron top level source directory.
""")
sys.exit()
sys.exit(1)


def _start_platform():
Expand Down Expand Up @@ -266,8 +270,8 @@ def func(*args, **kwargs):
print('Configuring {}.'.format(agent_dir))
config = config_func(*args, **kwargs)
_update_config_file()
#TODO: Optimize long vcfg install times
#TODO: (potentially only starting the platform once per vcfg)
# TODO: Optimize long vcfg install times
# TODO: (potentially only starting the platform once per vcfg)

if use_active in n:
_start_platform()
Expand Down Expand Up @@ -375,6 +379,7 @@ def set_dependencies(requirement):
subprocess.check_call(cmds)
return


def _create_web_certs():
global config_opts
"""
Expand Down Expand Up @@ -527,13 +532,15 @@ def do_instance_name():
instance_name = new_instance_name
config_opts['instance-name'] = '"{}"'.format(instance_name)


def do_web_enabled_rmq(vhome):
global config_opts

# Full implies that it will have a port on it as well. Though if it's
# not in the address that means that we haven't set it up before.
full_bind_web_address = config_opts.get('bind-web-address',
'https://' + get_hostname())
full_bind_web_address = config_opts.get(
'bind-web-address',
'https://' + get_hostname())

parsed = urlparse(full_bind_web_address)

Expand Down Expand Up @@ -575,7 +582,6 @@ def do_web_enabled_rmq(vhome):
def do_web_enabled_zmq(vhome):
global config_opts


# Full implies that it will have a port on it as well. Though if it's
# not in the address that means that we haven't set it up before.
full_bind_web_address = config_opts.get('bind-web-address',
Expand Down Expand Up @@ -892,7 +898,6 @@ def wizard():

# Start true configuration here.
volttron_home = get_home()
confirm_volttron_home()
_load_config()
_update_config_file()
if use_active in n:
Expand Down Expand Up @@ -993,11 +998,113 @@ def wizard():
if response in y:
do_listener()


def read_agent_configs_from_store(store_source, path=True):
if path:
with open(store_source) as f:
store = parse_json_config(f.read())
else:
store = store_source
return store


def update_configs_in_store(args_dict):

vhome = get_home()
try:
metadata_dict = load_yml_or_json(args_dict['metadata_file'])
except Exception as e:
print(f"Invalid metadata file: {args_dict['metadata_file']}: {e}")
exit(1)
for vip_id in metadata_dict:
configs = metadata_dict[vip_id]
if isinstance(configs, dict):
# only single config for this vip id
configs = [configs]
if not isinstance(configs, list):
print(
f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. "
f"Got type {type(configs)}")
_exit_with_metadata_error()

configs_updated = False
agent_store_path = os.path.join(vhome, "configuration_store", vip_id+".store")
if os.path.isfile(agent_store_path):
# load current store configs as python object for comparison
store_configs = read_agent_configs_from_store(agent_store_path)
else:
store_configs = dict()

for config_dict in configs:
if not isinstance(config_dict, dict):
print(f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. "
f"Got type {type(config_dict)}")
_exit_with_metadata_error()

config_name = config_dict.get("config-name", "config")
config_type = config_dict.get("config-type", "json")
config = config_dict.get("config")
if config is None:
print(f"No config entry found for vip-id {vip_id} and config-name {config_name}")
_exit_with_metadata_error()

# If there is config validate it
# Check if config is file path
if isinstance(config, str) and os.path.isfile(config):
raw_data = open(config).read()
# try loading it into appropriate python object to validate if file content and config-type match
processed_data = process_raw_config(raw_data, config_type)
elif isinstance(config, str) and config_type == 'raw':
raw_data = config
processed_data = config
else:
if not isinstance(config, (list, dict)):
processed_data = raw_data = None
print('Value for key "config" should be one of the following: \n'
'1. filepath \n'
'2. string with "config-type" set to "raw" \n'
'3. a dictionary \n'
'4. list ')
_exit_with_metadata_error()
else:
processed_data = config
raw_data = jsonapi.dumps(processed_data)

current = store_configs.get(config_name)

if not current or process_raw_config(current.get('data'), current.get('type')) != processed_data:
store_configs[config_name] = dict()
store_configs[config_name]['data'] = raw_data
store_configs[config_name]['type'] = config_type
store_configs[config_name]['modified'] = format_timestamp(get_aware_utc_now())
configs_updated = True

# All configs processed for current vip-id
# if there were updates write the new configs to file
if configs_updated:
os.makedirs(os.path.dirname(agent_store_path), exist_ok=True)
with open(agent_store_path, 'w+') as f:
json.dump(store_configs, f)


def _exit_with_metadata_error():
print('''Metadata file should be of the format:
{ "vip-id": [
{
"config-name": "optional. name. defaults to config",
"config": "json config or config file name",
"config-type": "optional. type of config. defaults to json"
}, ...
],...
}''')
exit(1)


def process_rmq_inputs(args_dict, instance_name=None):
#print(f"args_dict:{args_dict}, args")
if not is_rabbitmq_available():
raise RuntimeError("Rabbitmq Dependencies not installed please run python bootstrap.py --rabbitmq")
confirm_volttron_home()

vhome = get_home()

if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'):
Expand Down Expand Up @@ -1055,34 +1162,62 @@ def main():
parser.add_argument('--vhome', help="Path to volttron home")
parser.add_argument('--instance-name', dest='instance_name', help="Name of this volttron instance")
parser.set_defaults(is_rabbitmq=False)
parser.set_defaults(config_update=False)
group = parser.add_mutually_exclusive_group()

agent_list = '\n\t' + '\n\t'.join(sorted(available_agents.keys()))
group.add_argument('--list-agents', action='store_true', dest='list_agents',
help='list configurable agents{}'.format(agent_list))
rabbitmq_parser = parser.add_subparsers(title='rabbitmq',
metavar='',
dest='parser_name')
single_parser = rabbitmq_parser.add_parser('rabbitmq', help='Configure rabbitmq for single instance, '
'federation, or shovel either based on '
'configuration file in yml format or providing '
'details when prompted. \nUsage: vcfg rabbitmq '
'single|federation|shovel --config <rabbitmq config '
'file> --max-retries <attempt number>]')
single_parser.add_argument('installation-type', default='single', help='Rabbitmq option for installation. Installation type can be single|federation|shovel')
subparsers = parser.add_subparsers(dest="cmd")
single_parser = subparsers.add_parser('rabbitmq', help='Configure rabbitmq for single instance, '
'federation, or shovel either based on '
'configuration file in yml format or providing '
'details when prompted. \nUsage: vcfg rabbitmq '
'single|federation|shovel --config <rabbitmq config '
'file> --max-retries <attempt number>]')
single_parser.add_argument('installation-type', default='single',
help='Rabbitmq option for installation. '
'Installation type can be single|federation|shovel')
single_parser.add_argument('--max-retries', help='Optional Max retry attempt', type=int, default=12)
single_parser.add_argument('--config', help='Optional path to rabbitmq config yml', type=str)
single_parser.set_defaults(is_rabbitmq=True)

group.add_argument('--agent', nargs='+',
help='configure listed agents')
help='configure listed agents')

group.add_argument('--agent-isolation-mode', action='store_true', dest='agent_isolation_mode',
help='Require that agents run with their own users (this requires running '
'scripts/secure_user_permissions.sh as sudo)')
config_store_parser = subparsers.add_parser("update-config-store",
help="Update one or more config entries for one more agents")
config_store_parser.set_defaults(config_update=True)
# start with just a metadata file support.
# todo - add support vip-id, directory
# vip-id, file with multiple configs etc.
#config_arg_group = config_store_parser.add_mutually_exclusive_group()
#meta_group = config_arg_group.add_mutually_exclusive_group()
config_store_parser.add_argument('--metadata-file', required=True,
help='metadata file containing details of vip id, '
'optional config name(defaults to "config"),'
'config content, '
'and optional config type(defaults to json). Format:'
'\n'
'{ "vip-id": ['
' { '
' "config-name": "optional. name. defaults to config'
' "config": "json config or config file name", '
' "config-type": "optional. type of config. defaults to json"'
' }, ...'
' ],...'
'}')

# single_agent_group = config_arg_group.add_mutually_exclusive_group()
# single_agent_group.add_argument("--vip-id",
# help='vip-identity of the agent for which config store should be updated')
# single_agent_group.add_argument("--config-path",
# help="json file containing configs or directory containing config files")

args = parser.parse_args()

verbose = args.verbose
# Protect against configuration of base logger when not the "main entry point"
if verbose:
Expand All @@ -1094,8 +1229,18 @@ def main():
if args.vhome:
set_home(args.vhome)
prompt_vhome = False

confirm_volttron_home()
# if not args.rabbitmq or args.rabbitmq[0] in ["single"]:
fail_if_instance_running(args)
if args.agent:
message = "The VOLTTRON Instance is currently running. " \
"Installing agents to an active instance may overwrite currently installed "\
"and active agents on the platform, causing undesirable behavior. "
fail_if_instance_running(message)
if args.config_update:
message = f"VOLTTRON is running using at {get_home()}, " \
"you can add/update single configuration using vctl config command."
fail_if_instance_running(message, prompt=False)
fail_if_not_in_src_root()
if use_active in n:
atexit.register(_cleanup_on_exit)
Expand All @@ -1110,6 +1255,8 @@ def main():
_update_config_file()
elif args.is_rabbitmq:
process_rmq_inputs(vars(args))
elif args.config_update:
update_configs_in_store(vars(args))
elif not args.agent:
wizard()

Expand All @@ -1121,8 +1268,6 @@ def main():
print('"{}" not configurable with this tool'.format(agent))
else:
valid_agents = True
if valid_agents:
confirm_volttron_home()

# Configure agents
for agent in args.agent:
Expand Down

0 comments on commit 18b03e9

Please sign in to comment.