From da18bd1c56aa022cd674434cf37f0986c319c362 Mon Sep 17 00:00:00 2001
From: Mohamed Mehdi BEN AISSA
 <diode-ben-aissa.consultant@dgfip.finances.gouv.fr>
Date: Wed, 17 Jul 2024 15:10:38 +0200
Subject: [PATCH] feat(spark2|3): add ha support for spark-hs

---
 playbooks/files                               | 1 +
 playbooks/spark3_kerberos_install.yml         | 1 -
 tdp_vars_defaults/knox/knox.yml               | 6 ++++--
 tdp_vars_defaults/spark/spark.yml             | 2 +-
 tdp_vars_defaults/spark3/spark3.yml           | 2 +-
 tdp_vars_defaults/tdp-cluster/tdp-cluster.yml | 2 ++
 topology.ini                                  | 5 ++---
 7 files changed, 11 insertions(+), 8 deletions(-)
 create mode 120000 playbooks/files

diff --git a/playbooks/files b/playbooks/files
new file mode 120000
index 00000000..5f2042f1
--- /dev/null
+++ b/playbooks/files
@@ -0,0 +1 @@
+../../../../../../files
\ No newline at end of file
diff --git a/playbooks/spark3_kerberos_install.yml b/playbooks/spark3_kerberos_install.yml
index b229fd9d..21be7fce 100644
--- a/playbooks/spark3_kerberos_install.yml
+++ b/playbooks/spark3_kerberos_install.yml
@@ -13,7 +13,6 @@
         name: tosit.tdp.spark.historyserver
         tasks_from: kerberos
     - ansible.builtin.meta: clear_facts # noqa unnamed-task
-
 - name: Spark3 Kerberos Client install
   hosts: spark3_client
   strategy: linear
diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml
index 1d2d2c9b..7d105e2e 100644
--- a/tdp_vars_defaults/knox/knox.yml
+++ b/tdp_vars_defaults/knox/knox.yml
@@ -157,11 +157,13 @@ tdpldap_services:
     location: /ws
     port: "{{ yarn_rm_https_port }}"
   SPARKHISTORYUI:
-    hosts: "{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
+    hosts: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}"
     port: "{{ spark_hs_https_port }}"
+    scheme: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('scheme') }}://{% endif %}"
   SPARK3HISTORYUI:
-    hosts: "{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
+    hosts: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}"
     port: "{{ spark3_hs_https_port}}"
+    scheme: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('scheme') }}://{% endif %}"
   WEBHBASE:
     hosts: "{{ groups['hbase_rest'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
     port: "{{ hbase_rest_client_port }}"
diff --git a/tdp_vars_defaults/spark/spark.yml b/tdp_vars_defaults/spark/spark.yml
index 84b926ac..38472234 100644
--- a/tdp_vars_defaults/spark/spark.yml
+++ b/tdp_vars_defaults/spark/spark.yml
@@ -70,7 +70,7 @@ spark_truststore_location: /etc/ssl/certs/truststore.jks
 spark_truststore_password: Truststore123!
 
 # Spark History Server kerberos
-spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}"
+spark_ui_spnego_principal: "*"
 spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab
 
 # spark-defaults.conf - common
diff --git a/tdp_vars_defaults/spark3/spark3.yml b/tdp_vars_defaults/spark3/spark3.yml
index 88a3b704..260efcb9 100644
--- a/tdp_vars_defaults/spark3/spark3.yml
+++ b/tdp_vars_defaults/spark3/spark3.yml
@@ -71,7 +71,7 @@ hadoop_credentials_properties:
     value: '{{ spark_keystore_password }}'
 
 # Spark History Server kerberos
-spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}"
+spark_ui_spnego_principal: "*"
 spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab
 
 # spark-defaults.conf - common
diff --git a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml
index 8aaf4c7d..cacfb351 100644
--- a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml
+++ b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml
@@ -252,3 +252,5 @@ ldap:
 #############################
 
 # ranger_ha_address: "http[s]://dns_alias:port"
+# spark2_hs_ha_address: "http[s]://dns_alias:port"
+# spark3_hs_ha_address: "http[s]://dns_alias:port"
diff --git a/topology.ini b/topology.ini
index 709ee77d..4a45e9b5 100644
--- a/topology.ini
+++ b/topology.ini
@@ -97,12 +97,14 @@ master3
 edge
 
 [spark_hs:children]
+master2
 master3
 
 [spark_client:children]
 edge
 
 [spark3_hs:children]
+master2
 master3
 
 [spark3_client:children]
@@ -111,9 +113,6 @@ edge
 [knox:children]
 edge
 
-[spnego_ha:children]
-ranger_admin
-
 # Section Postgresql_client from tdp_prerequisites 
 [postgresql_client:children]
 ranger_admin