From dc19e87035a9252c567eded573c8b300a03196f8 Mon Sep 17 00:00:00 2001 From: Staroon Date: Thu, 7 Jun 2018 14:59:30 +0800 Subject: [PATCH] Change hadoop template, make it more compatible. --- hadoop-template/zabbix-hadoop.py | 95 ++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/hadoop-template/zabbix-hadoop.py b/hadoop-template/zabbix-hadoop.py index cca2e64..324c2ad 100644 --- a/hadoop-template/zabbix-hadoop.py +++ b/hadoop-template/zabbix-hadoop.py @@ -10,13 +10,16 @@ # Generate URL # --------------------------------- # This function converts the servername to URL which we need to query. -def get_url(server_name, listen_port): + + +def get_url(server_name, listen_port, topic_name): """ Generating URL to get the information from namenode/namenode :param server_name: :param listen_port: + :param topic_name: :return: """ @@ -28,7 +31,11 @@ def get_url(server_name, listen_port): print("Pass valid Hostname") exit() - URL = "http://"+server_name+":"+str(listen_port)+"/jmx?qry=Hadoop:*" + if not topic_name: + topic_name = "*" + + URL = "http://"+server_name+":" + \ + str(listen_port)+"/jmx?qry=Hadoop:"+topic_name return URL @@ -47,54 +54,69 @@ def load_url_as_dictionary(url): return json.load(urllib.urlopen(url)) -def json_processing(json_dict, monitor_type): +def json_processing(server_name, listen_port, monitor_type): namenode_dict = {} + jvm_json = load_url_as_dictionary( + get_url(server_name, listen_port, "service=NameNode,name=JvmMetrics")) + nns_json = load_url_as_dictionary( + get_url(server_name, listen_port, "service=NameNode,name=NameNodeStatus")) + fsns_json = load_url_as_dictionary( + get_url(server_name, listen_port, "service=NameNode,name=FSNamesystemState")) + nninfo_json = load_url_as_dictionary( + get_url(server_name, listen_port, "service=NameNode,name=NameNodeInfo")) + if "NN" == monitor_type: # JvmMetrics - namenode_dict['heap_memory_used'] = json_dict['beans'][0]['MemHeapUsedM'] - namenode_dict['heap_memory'] = json_dict['beans'][0]['MemHeapCommittedM'] - namenode_dict['max_heap_memory'] = json_dict['beans'][0]['MemHeapMaxM'] - namenode_dict['non_heap_memory_used'] = json_dict['beans'][0]['MemNonHeapUsedM'] - namenode_dict['commited_non_heap_memory'] = json_dict['beans'][0]['MemNonHeapCommittedM'] - namenode_dict['all_memory_used'] = float(json_dict['beans'][0]['MemHeapUsedM']) + float(json_dict['beans'][0]['MemNonHeapUsedM']) - + namenode_dict['heap_memory_used'] = jvm_json['beans'][0]['MemHeapUsedM'] + namenode_dict['heap_memory'] = jvm_json['beans'][0]['MemHeapCommittedM'] + namenode_dict['max_heap_memory'] = jvm_json['beans'][0]['MemHeapMaxM'] + namenode_dict['non_heap_memory_used'] = jvm_json['beans'][0]['MemNonHeapUsedM'] + namenode_dict['commited_non_heap_memory'] = jvm_json['beans'][0]['MemNonHeapCommittedM'] + namenode_dict['all_memory_used'] = float( + jvm_json['beans'][0]['MemHeapUsedM']) + float(jvm_json['beans'][0]['MemNonHeapUsedM']) + # NameNodeStatus - namenode_dict['namenode_state'] = json_dict['beans'][2]['State'] + namenode_dict['namenode_state'] = nns_json['beans'][0]['State'] # FSNamesystemState - timeStamp = float(str(json_dict['beans'][9]['BlockDeletionStartTime'])[:10]) + timeStamp = float( + str(fsns_json['beans'][0]['BlockDeletionStartTime'])[:10]) timeArray = time.localtime(timeStamp) - namenode_dict['start_time'] = time.strftime('%Y-%m-%d %H:%M:%S', timeArray) + namenode_dict['start_time'] = time.strftime( + '%Y-%m-%d %H:%M:%S', timeArray) # NameNodeInfo - namenode_dict['hadoop_version'] = str(json_dict['beans'][4]['Version']).split(',')[0] + namenode_dict['hadoop_version'] = str( + nninfo_json['beans'][0]['Version']).split(',')[0] return namenode_dict if "DFS" == monitor_type: # FSNamesystemState - namenode_dict['live_nodes'] = json_dict['beans'][9]['NumLiveDataNodes'] - namenode_dict['dead_nodes'] = json_dict['beans'][9]['NumDeadDataNodes'] - namenode_dict['decommissioning_nodes'] = json_dict['beans'][9]['NumDecommissioningDataNodes'] - namenode_dict['under_replicated_blocks'] = json_dict['beans'][9]['UnderReplicatedBlocks'] + namenode_dict['live_nodes'] = fsns_json['beans'][0]['NumLiveDataNodes'] + namenode_dict['dead_nodes'] = fsns_json['beans'][0]['NumDeadDataNodes'] + namenode_dict['decommissioning_nodes'] = fsns_json['beans'][0]['NumDecommissioningDataNodes'] + namenode_dict['under_replicated_blocks'] = fsns_json['beans'][0]['UnderReplicatedBlocks'] # NameNodeInfo - namenode_dict['files_and_directorys'] = json_dict['beans'][4]['TotalFiles'] - namenode_dict['blocks'] = json_dict['beans'][4]['TotalBlocks'] - namenode_dict['configured_capacity'] = json_dict['beans'][4]['Total'] - namenode_dict['dfs_used'] = json_dict['beans'][4]['Used'] - namenode_dict['dfs_used_persent'] = min_value_display(float(json_dict['beans'][4]['PercentUsed'])) - namenode_dict['non_dfs_used'] = json_dict['beans'][4]['NonDfsUsedSpace'] - namenode_dict['dfs_remaining'] = json_dict['beans'][4]['Free'] - namenode_dict['dfs_remaining_persent'] = min_value_display(float(json_dict['beans'][4]['PercentRemaining'])) - datanodes_usages = json.loads(json_dict['beans'][4]['NodeUsage']) + namenode_dict['files_and_directorys'] = nninfo_json['beans'][0]['TotalFiles'] + namenode_dict['blocks'] = nninfo_json['beans'][0]['TotalBlocks'] + namenode_dict['configured_capacity'] = nninfo_json['beans'][0]['Total'] + namenode_dict['dfs_used'] = nninfo_json['beans'][0]['Used'] + namenode_dict['dfs_used_persent'] = min_value_display( + float(nninfo_json['beans'][0]['PercentUsed'])) + namenode_dict['non_dfs_used'] = nninfo_json['beans'][0]['NonDfsUsedSpace'] + namenode_dict['dfs_remaining'] = nninfo_json['beans'][0]['Free'] + namenode_dict['dfs_remaining_persent'] = min_value_display( + float(nninfo_json['beans'][0]['PercentRemaining'])) + datanodes_usages = json.loads(nninfo_json['beans'][0]['NodeUsage']) namenode_dict['min_datanodes_usages'] = datanodes_usages['nodeUsage']['min'] namenode_dict['median_datanodes_usages'] = datanodes_usages['nodeUsage']['median'] namenode_dict['max_datanodes_usages'] = datanodes_usages['nodeUsage']['max'] namenode_dict['stdDev_datanodes_usages'] = datanodes_usages['nodeUsage']['stdDev'] - live_nodes = json.loads(json_dict['beans'][4]['LiveNodes']) + live_nodes = json.loads(nninfo_json['beans'][0]['LiveNodes']) HIGH_CONST = 99999 LOW_CONST = -99999 @@ -110,7 +132,8 @@ def json_processing(json_dict, monitor_type): live_node_name = live_node_in_list capacity = float(live_nodes[live_node_in_list]['capacity']) / GB - node_remaining = float(min_value_display(float(live_nodes[live_node_in_list]['remaining']) / GB)) + node_remaining = float(min_value_display( + float(live_nodes[live_node_in_list]['remaining']) / GB)) node_remaining_persent = node_remaining / capacity * 100 if node_remaining > max_node_remaining: @@ -139,16 +162,20 @@ def json_processing(json_dict, monitor_type): ''' exit() + def write_data_to_file(json, file_path, hadoop_name_in_zabbix): txt_file = open(file_path, 'w+') for keys in json: - txt_file.writelines(hadoop_name_in_zabbix +' '+ str(keys) +' '+ str(json[keys]) + '\n') + txt_file.writelines(hadoop_name_in_zabbix + ' ' + + str(keys) + ' ' + str(json[keys]) + '\n') + def usage(): print ''' Usage: $SCRIPT_NAME ''' + def min_value_display(x): return '{0:.2f}'.format(x) @@ -163,13 +190,11 @@ def min_value_display(x): namenode_listen_port = sys.argv[2] file_path = sys.argv[3] nodename_in_zabbix = sys.argv[4] - monitor_type=sys.argv[5] + monitor_type = sys.argv[5] - url = get_url(namenode_host, namenode_listen_port) - json_as_dictionary = load_url_as_dictionary(url) - json_processed = json_processing(json_as_dictionary, monitor_type) + json_processed = json_processing( + namenode_host, namenode_listen_port, monitor_type) write_data_to_file(json_processed, file_path, nodename_in_zabbix) else: usage() -