Skip to content

Commit

Permalink
Change hadoop template, make it more compatible.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Staroon
Staroon committed Jun 7, 2018
1 parent b7cb2d8 commit dc19e87
Showing 1 changed file with 60 additions and 35 deletions.
95 changes: 60 additions & 35 deletions hadoop-template/zabbix-hadoop.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,16 @@
# Generate URL
# ---------------------------------
# This function converts the servername to URL which we need to query.
def get_url(server_name, listen_port):


def get_url(server_name, listen_port, topic_name):
"""
Generating URL to get the information
from namenode/namenode
:param server_name:
:param listen_port:
:param topic_name:
:return:
"""

Expand All @@ -28,7 +31,11 @@ def get_url(server_name, listen_port):
print("Pass valid Hostname")
exit()

URL = "http://"+server_name+":"+str(listen_port)+"/jmx?qry=Hadoop:*"
if not topic_name:
topic_name = "*"

URL = "http://"+server_name+":" + \
str(listen_port)+"/jmx?qry=Hadoop:"+topic_name
return URL


Expand All @@ -47,54 +54,69 @@ def load_url_as_dictionary(url):
return json.load(urllib.urlopen(url))


def json_processing(json_dict, monitor_type):
def json_processing(server_name, listen_port, monitor_type):

namenode_dict = {}

jvm_json = load_url_as_dictionary(
get_url(server_name, listen_port, "service=NameNode,name=JvmMetrics"))
nns_json = load_url_as_dictionary(
get_url(server_name, listen_port, "service=NameNode,name=NameNodeStatus"))
fsns_json = load_url_as_dictionary(
get_url(server_name, listen_port, "service=NameNode,name=FSNamesystemState"))
nninfo_json = load_url_as_dictionary(
get_url(server_name, listen_port, "service=NameNode,name=NameNodeInfo"))

if "NN" == monitor_type:
# JvmMetrics
namenode_dict['heap_memory_used'] = json_dict['beans'][0]['MemHeapUsedM']
namenode_dict['heap_memory'] = json_dict['beans'][0]['MemHeapCommittedM']
namenode_dict['max_heap_memory'] = json_dict['beans'][0]['MemHeapMaxM']
namenode_dict['non_heap_memory_used'] = json_dict['beans'][0]['MemNonHeapUsedM']
namenode_dict['commited_non_heap_memory'] = json_dict['beans'][0]['MemNonHeapCommittedM']
namenode_dict['all_memory_used'] = float(json_dict['beans'][0]['MemHeapUsedM']) + float(json_dict['beans'][0]['MemNonHeapUsedM'])

namenode_dict['heap_memory_used'] = jvm_json['beans'][0]['MemHeapUsedM']
namenode_dict['heap_memory'] = jvm_json['beans'][0]['MemHeapCommittedM']
namenode_dict['max_heap_memory'] = jvm_json['beans'][0]['MemHeapMaxM']
namenode_dict['non_heap_memory_used'] = jvm_json['beans'][0]['MemNonHeapUsedM']
namenode_dict['commited_non_heap_memory'] = jvm_json['beans'][0]['MemNonHeapCommittedM']
namenode_dict['all_memory_used'] = float(
jvm_json['beans'][0]['MemHeapUsedM']) + float(jvm_json['beans'][0]['MemNonHeapUsedM'])

# NameNodeStatus
namenode_dict['namenode_state'] = json_dict['beans'][2]['State']
namenode_dict['namenode_state'] = nns_json['beans'][0]['State']

# FSNamesystemState
timeStamp = float(str(json_dict['beans'][9]['BlockDeletionStartTime'])[:10])
timeStamp = float(
str(fsns_json['beans'][0]['BlockDeletionStartTime'])[:10])
timeArray = time.localtime(timeStamp)
namenode_dict['start_time'] = time.strftime('%Y-%m-%d %H:%M:%S', timeArray)
namenode_dict['start_time'] = time.strftime(
'%Y-%m-%d %H:%M:%S', timeArray)

# NameNodeInfo
namenode_dict['hadoop_version'] = str(json_dict['beans'][4]['Version']).split(',')[0]
namenode_dict['hadoop_version'] = str(
nninfo_json['beans'][0]['Version']).split(',')[0]

return namenode_dict

if "DFS" == monitor_type:
# FSNamesystemState
namenode_dict['live_nodes'] = json_dict['beans'][9]['NumLiveDataNodes']
namenode_dict['dead_nodes'] = json_dict['beans'][9]['NumDeadDataNodes']
namenode_dict['decommissioning_nodes'] = json_dict['beans'][9]['NumDecommissioningDataNodes']
namenode_dict['under_replicated_blocks'] = json_dict['beans'][9]['UnderReplicatedBlocks']
namenode_dict['live_nodes'] = fsns_json['beans'][0]['NumLiveDataNodes']
namenode_dict['dead_nodes'] = fsns_json['beans'][0]['NumDeadDataNodes']
namenode_dict['decommissioning_nodes'] = fsns_json['beans'][0]['NumDecommissioningDataNodes']
namenode_dict['under_replicated_blocks'] = fsns_json['beans'][0]['UnderReplicatedBlocks']

# NameNodeInfo
namenode_dict['files_and_directorys'] = json_dict['beans'][4]['TotalFiles']
namenode_dict['blocks'] = json_dict['beans'][4]['TotalBlocks']
namenode_dict['configured_capacity'] = json_dict['beans'][4]['Total']
namenode_dict['dfs_used'] = json_dict['beans'][4]['Used']
namenode_dict['dfs_used_persent'] = min_value_display(float(json_dict['beans'][4]['PercentUsed']))
namenode_dict['non_dfs_used'] = json_dict['beans'][4]['NonDfsUsedSpace']
namenode_dict['dfs_remaining'] = json_dict['beans'][4]['Free']
namenode_dict['dfs_remaining_persent'] = min_value_display(float(json_dict['beans'][4]['PercentRemaining']))
datanodes_usages = json.loads(json_dict['beans'][4]['NodeUsage'])
namenode_dict['files_and_directorys'] = nninfo_json['beans'][0]['TotalFiles']
namenode_dict['blocks'] = nninfo_json['beans'][0]['TotalBlocks']
namenode_dict['configured_capacity'] = nninfo_json['beans'][0]['Total']
namenode_dict['dfs_used'] = nninfo_json['beans'][0]['Used']
namenode_dict['dfs_used_persent'] = min_value_display(
float(nninfo_json['beans'][0]['PercentUsed']))
namenode_dict['non_dfs_used'] = nninfo_json['beans'][0]['NonDfsUsedSpace']
namenode_dict['dfs_remaining'] = nninfo_json['beans'][0]['Free']
namenode_dict['dfs_remaining_persent'] = min_value_display(
float(nninfo_json['beans'][0]['PercentRemaining']))
datanodes_usages = json.loads(nninfo_json['beans'][0]['NodeUsage'])
namenode_dict['min_datanodes_usages'] = datanodes_usages['nodeUsage']['min']
namenode_dict['median_datanodes_usages'] = datanodes_usages['nodeUsage']['median']
namenode_dict['max_datanodes_usages'] = datanodes_usages['nodeUsage']['max']
namenode_dict['stdDev_datanodes_usages'] = datanodes_usages['nodeUsage']['stdDev']
live_nodes = json.loads(json_dict['beans'][4]['LiveNodes'])
live_nodes = json.loads(nninfo_json['beans'][0]['LiveNodes'])

HIGH_CONST = 99999
LOW_CONST = -99999
Expand All @@ -110,7 +132,8 @@ def json_processing(json_dict, monitor_type):
live_node_name = live_node_in_list
capacity = float(live_nodes[live_node_in_list]['capacity']) / GB

node_remaining = float(min_value_display(float(live_nodes[live_node_in_list]['remaining']) / GB))
node_remaining = float(min_value_display(
float(live_nodes[live_node_in_list]['remaining']) / GB))
node_remaining_persent = node_remaining / capacity * 100

if node_remaining > max_node_remaining:
Expand Down Expand Up @@ -139,16 +162,20 @@ def json_processing(json_dict, monitor_type):
'''
exit()


def write_data_to_file(json, file_path, hadoop_name_in_zabbix):
txt_file = open(file_path, 'w+')
for keys in json:
txt_file.writelines(hadoop_name_in_zabbix +' '+ str(keys) +' '+ str(json[keys]) + '\n')
txt_file.writelines(hadoop_name_in_zabbix + ' ' +
str(keys) + ' ' + str(json[keys]) + '\n')


def usage():
print '''
Usage: $SCRIPT_NAME <hadoop_host> <hadoop_port> <file_path> <zabbix_server_name> <monitor_type>
'''


def min_value_display(x):
return '{0:.2f}'.format(x)

Expand All @@ -163,13 +190,11 @@ def min_value_display(x):
namenode_listen_port = sys.argv[2]
file_path = sys.argv[3]
nodename_in_zabbix = sys.argv[4]
monitor_type=sys.argv[5]
monitor_type = sys.argv[5]

url = get_url(namenode_host, namenode_listen_port)
json_as_dictionary = load_url_as_dictionary(url)
json_processed = json_processing(json_as_dictionary, monitor_type)
json_processed = json_processing(
namenode_host, namenode_listen_port, monitor_type)
write_data_to_file(json_processed, file_path, nodename_in_zabbix)

else:
usage()

0 comments on commit dc19e87

Please sign in to comment.