Add ConfigProvider SPI and K8s implementation to load ConfigMap resources (#80)

* Load configMap properties at start up

* Add config provider to clean up dependencies

* Address feedback - change service loader interface, removing system props, and load configs per driver

* Move field expansion out of SPI

Author: jogrogan

Makefile

@@ -16,7 +16,7 @@ clean:
 	./gradlew clean
 
 deploy-config:
-	kubectl create configmap hoptimator-configmap --from-file=model.yaml=test-model.yaml --dry-run=client -o yaml | kubectl apply -f -
+	kubectl apply -f ./deploy/config/hoptimator-configmap.yaml
 
 undeploy-config:
 	kubectl delete configmap hoptimator-configmap || echo "skipping"

deploy/config/hoptimator-configmap.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hoptimator-configmap
+data:
+  # https://kubernetes.io/docs/concepts/configuration/configmap/
+
+  # property-like keys; each key maps to a simple value
+  player_initial_lives: "3"
+  ui_properties_file_name: "user-interface.properties"
+
+  # file-like keys
+  game.properties: |
+    enemy.types=aliens,monsters
+    player.maximum-lives=5
+  user-interface.properties: |
+    color.good=purple
+    color.bad=yellow
+    allow.textmode=true

hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java

@@ -0,0 +1,8 @@
+package com.linkedin.hoptimator;
+
+import java.util.Properties;
+
+public interface ConfigProvider {
+
+  Properties loadConfig() throws Exception;
+}

hoptimator-cli/build.gradle

@@ -6,7 +6,6 @@ plugins {
 }
 
 dependencies {
-  implementation project(':hoptimator-jdbc')
   implementation project(':hoptimator-api')
   implementation project(':hoptimator-avro')
   implementation project(':hoptimator-demodb')

hoptimator-jdbc/build.gradle

@@ -14,23 +14,40 @@ dependencies {
   testFixturesImplementation libs.calcite.core
   testFixturesImplementation project(':hoptimator-api')
   testFixturesImplementation project(':hoptimator-util')
+  testFixturesImplementation(platform('org.junit:junit-bom:5.11.3'))
+  testFixturesImplementation 'org.junit.jupiter:junit-jupiter'
 
   testRuntimeOnly project(':hoptimator-demodb')
-	testFixturesImplementation(platform('org.junit:junit-bom:5.11.3'))
-	testImplementation(platform('org.junit:junit-bom:5.11.3'))
-	testImplementation 'org.junit.jupiter:junit-jupiter'
-	testFixturesImplementation 'org.junit.jupiter:junit-jupiter'
+  testImplementation(platform('org.junit:junit-bom:5.11.3'))
+  testImplementation 'org.junit.jupiter:junit-jupiter'
   testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
   testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }
 
 test {
-	useJUnitPlatform()
+  useJUnitPlatform {
+    excludeTags 'integration'
+  }
 	testLogging {
 		events "passed", "skipped", "failed"
 	}
 }
 
+tasks.register('intTest', Test) {
+  description = 'Runs integration tests.'
+  group = 'verification'
+
+  shouldRunAfter test
+
+  useJUnitPlatform {
+    includeTags 'integration'
+  }
+
+  testLogging {
+    events "passed", "skipped", "failed"
+  }
+}
+
 java {
   sourceCompatibility = JavaVersion.VERSION_11
   targetCompatibility = JavaVersion.VERSION_11

hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java

@@ -23,7 +23,7 @@
 public class HoptimatorDriver extends Driver {
 
   public HoptimatorDriver() {
-    super(() -> new Prepare());
+    super(Prepare::new);
   }
 
   static {

hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java

@@ -0,0 +1,12 @@
+package com.linkedin.hoptimator.jdbc;
+
+import java.util.Properties;
+
+import com.linkedin.hoptimator.ConfigProvider;
+
+public class SystemPropertiesConfigProvider implements ConfigProvider {
+
+  public Properties loadConfig() {
+    return System.getProperties();
+  }
+}

hoptimator-jdbc/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider

@@ -0,0 +1 @@
+com.linkedin.hoptimator.jdbc.SystemPropertiesConfigProvider

hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java

@@ -1,5 +1,7 @@
 package com.linkedin.hoptimator.k8s;
 
+import io.kubernetes.client.openapi.models.V1ConfigMap;
+import io.kubernetes.client.openapi.models.V1ConfigMapList;
 import io.kubernetes.client.openapi.models.V1Namespace;
 import io.kubernetes.client.openapi.models.V1NamespaceList;
 import io.kubernetes.client.openapi.models.V1Secret;
@@ -24,6 +26,9 @@ public final class K8sApiEndpoints {
       new K8sApiEndpoint<>("Namespace", "", "v1", "namespaces", true, V1Namespace.class, V1NamespaceList.class);
   public static final K8sApiEndpoint<V1Secret, V1SecretList> SECRETS =
       new K8sApiEndpoint<>("Secret", "", "v1", "secrets", false, V1Secret.class, V1SecretList.class);
+  public static final K8sApiEndpoint<V1ConfigMap, V1ConfigMapList> CONFIG_MAPS =
+      new K8sApiEndpoint<>("ConfigMap", "", "v1", "configmaps", false,
+          V1ConfigMap.class, V1ConfigMapList.class);
 
   // Hoptimator custom resources
   public static final K8sApiEndpoint<V1alpha1Database, V1alpha1DatabaseList> DATABASES =

hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java

@@ -0,0 +1,29 @@
+package com.linkedin.hoptimator.k8s;
+
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+import io.kubernetes.client.openapi.models.V1ConfigMap;
+import io.kubernetes.client.openapi.models.V1ConfigMapList;
+
+import com.linkedin.hoptimator.ConfigProvider;
+
+
+public class K8sConfigProvider implements ConfigProvider {
+
+  public static final String HOPTIMATOR_CONFIG_MAP = "hoptimator-configmap";
+
+  public Properties loadConfig() throws SQLException {
+    Map<String, String> topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP);
+    Properties p = new Properties();
+    p.putAll(topLevelConfigs);
+    return p;
+  }
+
+  // Load top-level config map properties
+  private Map<String, String> loadTopLevelConfig(String configMapName) throws SQLException {
+    K8sApi<V1ConfigMap, V1ConfigMapList> configMapApi = new K8sApi<>(K8sContext.currentContext(), K8sApiEndpoints.CONFIG_MAPS);
+    return configMapApi.get(configMapName).getData();
+  }
+}

hoptimator-k8s/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider

@@ -0,0 +1 @@
+com.linkedin.hoptimator.k8s.K8sConfigProvider

hoptimator-kafka/build.gradle

@@ -3,6 +3,7 @@ plugins {
 }
 
 dependencies {
+  implementation project(':hoptimator-util')
   implementation libs.calcite.core
   implementation libs.kafka.clients
 
@@ -15,8 +16,8 @@ dependencies {
   testRuntimeOnly project(':hoptimator-k8s')
   testRuntimeOnly project(':hoptimator-kafka')
   testImplementation(testFixtures(project(':hoptimator-jdbc')))
-	testImplementation(platform('org.junit:junit-bom:5.11.3'))
-	testImplementation 'org.junit.jupiter:junit-jupiter'
+  testImplementation(platform('org.junit:junit-bom:5.11.3'))
+  testImplementation 'org.junit.jupiter:junit-jupiter'
   testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
   testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }

hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDriver.java

@@ -11,6 +11,8 @@
 import org.apache.calcite.jdbc.Driver;
 import org.apache.calcite.schema.SchemaPlus;
 
+import com.linkedin.hoptimator.util.ConfigService;
+
 
 /** JDBC driver for Kafka topics. */
 public class KafkaDriver extends Driver {
@@ -34,7 +36,9 @@ public Connection connect(String url, Properties props) throws SQLException {
     if (!url.startsWith(getConnectStringPrefix())) {
       return null;
     }
-    Properties properties = ConnectStringParser.parse(url.substring(getConnectStringPrefix().length()));
+    // Connection string properties are given precedence over config properties
+    Properties properties = ConfigService.config();
+    properties.putAll(ConnectStringParser.parse(url.substring(getConnectStringPrefix().length())));
     try {
       Connection connection = super.connect(url, props);
       if (connection == null) {

hoptimator-util/build.gradle

@@ -8,8 +8,8 @@ dependencies {
   implementation libs.calcite.core
 
   testImplementation(testFixtures(project(':hoptimator-jdbc')))
-	testImplementation(platform('org.junit:junit-bom:5.11.3'))
-	testImplementation 'org.junit.jupiter:junit-jupiter'
+  testImplementation(platform('org.junit:junit-bom:5.11.3'))
+  testImplementation 'org.junit.jupiter:junit-jupiter'
   testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
   testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java

@@ -0,0 +1,44 @@
+package com.linkedin.hoptimator.util;
+
+import java.io.StringReader;
+import java.util.Properties;
+import java.util.ServiceLoader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.linkedin.hoptimator.ConfigProvider;
+
+public final class ConfigService {
+
+  private static final Logger log = LoggerFactory.getLogger(ConfigService.class);
+
+  private ConfigService() {
+  }
+
+  // Loads top level configs and expands input fields as file-like properties
+  // Ex:
+  //  log.properties: |
+  //    level=INFO
+  public static Properties config(String... expansionFields) {
+    ServiceLoader<ConfigProvider> loader = ServiceLoader.load(ConfigProvider.class);
+    Properties properties = new Properties();
+    for (ConfigProvider provider : loader) {
+      try {
+        Properties loadedProperties = provider.loadConfig();
+        log.debug("Loaded properties={} from provider={}", loadedProperties, provider);
+        properties.putAll(loadedProperties);
+        for (String expansionField : expansionFields) {
+          if (loadedProperties == null || !loadedProperties.containsKey(expansionField)) {
+            log.warn("provider={} does not contain field={}", provider, expansionField);
+            continue;
+          }
+          properties.load(new StringReader(loadedProperties.getProperty(expansionField)));
+        }
+      } catch (Exception e) {
+        log.warn("Could not load properties for provider={}", provider, e);
+      }
+    }
+    return properties;
+  }
+}

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DataTypeUtils.java

@@ -3,6 +3,7 @@
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -80,7 +81,7 @@ private static RelDataType buildRecord(Node node, RelDataTypeFactory typeFactory
       return node.dataType;
     }
     RelDataTypeFactory.Builder builder = new RelDataTypeFactory.Builder(typeFactory);
-    for (LinkedHashMap.Entry<String, Node> child : node.children.entrySet()) {
+    for (Map.Entry<String, Node> child : node.children.entrySet()) {
       builder.add(child.getKey(), buildRecord(child.getValue(), typeFactory));
     }
     return builder.build();

hoptimator-venice/build.gradle

@@ -21,8 +21,8 @@ dependencies {
 
   testRuntimeOnly project(':hoptimator-k8s')
   testImplementation(testFixtures(project(':hoptimator-jdbc')))
-	testImplementation(platform('org.junit:junit-bom:5.11.3'))
-	testImplementation 'org.junit.jupiter:junit-jupiter'
+  testImplementation(platform('org.junit:junit-bom:5.11.3'))
+  testImplementation 'org.junit.jupiter:junit-jupiter'
   testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
   testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }

hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDriver.java

@@ -12,11 +12,13 @@
 import org.apache.calcite.jdbc.Driver;
 import org.apache.calcite.schema.SchemaPlus;
 
+import com.linkedin.hoptimator.util.ConfigService;
+
 
 /** JDBC driver for Venice stores. */
 public class VeniceDriver extends Driver {
-
   public static final String CATALOG_NAME = "VENICE";
+  public static final String CONFIG_NAME = "venice.config";
 
   static {
     new VeniceDriver().register();
@@ -37,7 +39,9 @@ public Connection connect(String url, Properties props) throws SQLException {
     if (!url.startsWith(getConnectStringPrefix())) {
       return null;
     }
-    Properties properties = ConnectStringParser.parse(url.substring(getConnectStringPrefix().length()));
+    // Connection string properties are given precedence over config properties
+    Properties properties = ConfigService.config(CONFIG_NAME);
+    properties.putAll(ConnectStringParser.parse(url.substring(getConnectStringPrefix().length())));
     String cluster = properties.getProperty("cluster");
     if (cluster == null) {
       throw new IllegalArgumentException("Missing required cluster property. Need: jdbc:venice://cluster=...");