feat(spark2|3): add ha support for spark-hs

playbooks/files

@@ -0,0 +1 @@
+../../../../../../files

playbooks/spark3_kerberos_install.yml

@@ -13,7 +13,6 @@
         name: tosit.tdp.spark.historyserver
         tasks_from: kerberos
     - ansible.builtin.meta: clear_facts # noqa unnamed-task
-
 - name: Spark3 Kerberos Client install
   hosts: spark3_client
   strategy: linear

tdp_vars_defaults/knox/knox.yml

@@ -157,11 +157,13 @@ tdpldap_services:
     location: /ws
     port: "{{ yarn_rm_https_port }}"
   SPARKHISTORYUI:
-    hosts: "{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
+    hosts: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}"
     port: "{{ spark_hs_https_port }}"
+    scheme: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('scheme') }}://{% endif %}"
   SPARK3HISTORYUI:
-    hosts: "{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
+    hosts: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}"
     port: "{{ spark3_hs_https_port}}"
+    scheme: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('scheme') }}://{% endif %}"
   WEBHBASE:
     hosts: "{{ groups['hbase_rest'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
     port: "{{ hbase_rest_client_port }}"

tdp_vars_defaults/spark/spark.yml

@@ -70,7 +70,7 @@ spark_truststore_location: /etc/ssl/certs/truststore.jks
 spark_truststore_password: Truststore123!
 
 # Spark History Server kerberos
-spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}"
+spark_ui_spnego_principal: "*"
 spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab
 
 # spark-defaults.conf - common

tdp_vars_defaults/spark3/spark3.yml

@@ -71,7 +71,7 @@ hadoop_credentials_properties:
     value: '{{ spark_keystore_password }}'
 
 # Spark History Server kerberos
-spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}"
+spark_ui_spnego_principal: "*"
 spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab
 
 # spark-defaults.conf - common

tdp_vars_defaults/tdp-cluster/tdp-cluster.yml

@@ -252,3 +252,5 @@ ldap:
 #############################
 
 # ranger_ha_address: "http[s]://dns_alias:port"
+# spark2_hs_ha_address: "http[s]://dns_alias:port"
+# spark3_hs_ha_address: "http[s]://dns_alias:port"

topology.ini

@@ -97,12 +97,14 @@ master3
 edge
 
 [spark_hs:children]
+master2
 master3
 
 [spark_client:children]
 edge
 
 [spark3_hs:children]
+master2
 master3
 
 [spark3_client:children]
@@ -111,9 +113,6 @@ edge
 [knox:children]
 edge
 
-[spnego_ha:children]
-ranger_admin
-
 # Section Postgresql_client from tdp_prerequisites 
 [postgresql_client:children]
 ranger_admin