Merge pull request #8 from innogames/tech_27700

Add support for draining LB Nodes

Author: TuxPowered42

.gitignore

@@ -2,3 +2,4 @@
 *.o
 obj/*
 testtool
+build

CMakeLists.txt

@@ -1,34 +1,106 @@
 cmake_minimum_required(VERSION 3.11)
 project(testtool)
+enable_testing()
 
 set(default_build_type "RelWithDebInfo")
 
 add_compile_options(
   "-Wall" "-pedantic" "-std=c++11"
 )
 
-include_directories(/usr/local/include)
-link_directories(/usr/local/lib)
+if (APPLE)
+  # Include packages installed from brew
+  set(OPENSSL_ROOT_DIR /usr/local/opt/openssl)
+  set(nlohmann_json_DIR /usr/local/opt/nlohmann-json/lib/cmake/nlohmann_json/)
+endif()
 
 set(Boost_USE_STATIC_LIBS ON)
-find_package(OpenSSL REQUIRED)
 find_package(Boost REQUIRED COMPONENTS system)
-find_package(PostgreSQL REQUIRED)
+list(APPEND _include_dirs ${Boost_INCLUDE_DIR})
+list(APPEND _libs ${Boost_LIBRARIES})
+
+find_package(fmt 3.3.0 REQUIRED)
+
 find_package(nlohmann_json 3.3.0 REQUIRED)
+if (APPLE)
+  # Does not provide include dir as it seems
+  list(APPEND _include_dirs /usr/local/opt/nlohmann-json/include)
+endif()
+list(APPEND _libs nlohmann_json::nlohmann_json)
 
-file(GLOB testtool_src "src/*.cpp")
-add_executable(testtool ${testtool_src})
-target_link_libraries(testtool
-    event.a event_core.a event_pthreads.a event_openssl.a
-    thr.a pq.a intl.a
-    fmt
-    ${OPENSSL_LIBRARIES}
-    ${Boost_LIBRARIES}
-    nlohmann_json::nlohmann_json
-)
+# find_package(libevent REQUIRED) # Does not come with cmake support?
+# Libraries will hopefully be in default directory
+if (APPLE)
+  list(APPEND _include_dirs /usr/local/opt/libevent/include)
+  list(APPEND _link_dirs /usr/local/opt/libevent/lib)
+endif()
+if(UNIX AND NOT APPLE AND NOT LINUX)
+  set(_link_dirs /usr/local/lib)
+  set(_include_dirs /usr/local/include)
+endif()
+list(APPEND _libs event.a event_core.a event_pthreads.a event_openssl.a)
+
+find_package(OpenSSL REQUIRED)
+list(APPEND _include_dirs ${OPENSSL_INCLUDE_DIR})
+list(APPEND _libs ${OPENSSL_LIBRARIES})
+
+find_package(PostgreSQL REQUIRED)
+list(APPEND _libs libpq.a)
+
+find_package(yaml-cpp REQUIRED)
+list(APPEND _include_dirs ${YAML_CPP_INCLUDE_DIR})
+list(APPEND _libs ${YAML_CPP_LIBRARIES})
+
+if(UNIX AND NOT APPLE AND NOT LINUX)
+  # Only on FreeBSD
+  list(APPEND _libs thr.a)
+  list(APPEND _libs libintl.a)
+endif()
+
+include_directories(${_include_dirs})
+link_directories(${_link_dirs})
+
+file(GLOB testtool_sources "src/*.cpp")
+file(GLOB testtool_headers "src/*.h")
+foreach(testtool_source ${testtool_sources})
+  string(REGEX REPLACE ".*/" "" testtool_library ${testtool_source})
+  add_library("${testtool_library}" OBJECT ${testtool_source} ${testool_headers})
+  list(APPEND testtool_libraries ${testtool_library})
+endforeach()
+
+add_executable(testtool)
+target_link_libraries(testtool ${_libs} ${testtool_libraries})
 install(TARGETS testtool DESTINATION sbin)
 install(
     PROGRAMS rc/testtool_freebsd
     DESTINATION etc/rc.d
     RENAME testtool
 )
+
+list(APPEND testtool_test_libraries ${testtool_libraries})
+list(FILTER testtool_test_libraries EXCLUDE REGEX "^(msg|pfctl_worker|pfctl|testtool).cpp$")
+
+find_package(GTest REQUIRED)
+include(GoogleTest)
+
+configure_file("tests/cmake_dirs.h.in" ${CMAKE_BINARY_DIR}/generated/cmake_dirs.h)
+include_directories(${CMAKE_BINARY_DIR}/generated ${CMAKE_SOURCE_DIR}/src)
+
+file(GLOB testtool_test_src "tests/*.cpp" "tests/*.h")
+add_executable(testtool_test ${testtool_test_src})
+add_dependencies(testtool_test ${testtool_test_libraries})
+gtest_discover_tests(testtool_test)
+
+target_link_libraries(testtool_test
+  GTest::GTest
+  GTest::Main 
+  ${testtool_test_libraries}
+  ${_libs}
+)
+
+# Create autoconf-compatible "check" commmand which also fixes
+# the issue of CMake not including dependency from test to test binaries
+add_custom_target(check
+ COMMAND ${CMAKE_CTEST_COMMAND}
+ DEPENDS testtool_test
+)

README.rst

@@ -31,9 +31,10 @@ HTTP and HTTPS
 
 The check sends the following HTTP request::
 
-	$hc_query HTTP/1.1
-	Host: $ip_addr
-	Connection: close
+  $hc_query HTTP/1.1
+  Host: $ip_addr
+  User-agent: testtool
+  Connection: close
 
 The request is sent to the specified port, or port 80 for `http` or port
 443 for `https`.  Health check port number has nothing to do with forwarded
@@ -51,8 +52,22 @@ Fails on:
 Type-specific attributes:
 
 * `hc_port`: Port number to connect to
-* `hc_query`: HTTP method and the URL (path) to test
-  (e.g. "HEAD /backend/lb_check.php")
+* `hc_query`: HTTP method and the URL (path) to test, like
+  "HEAD /backend/lb_check.php?pool={POOL_NAME}".
+  There are a few macros available which can be put into the query.
+  For expansion of a single IP address the address of the same address family
+  as the check is choosen. For expansion of multiple addresses both IPv4 and
+  IPv6 addresses will be sent.
+
+  - `{POOL_NAME}` - hostname of currently checked LB Pool
+  - `{POOL_ADDRESS}` - IP address of currently checked LB Pool
+  - `{NODE_NAME}` - hostname of currently checked LB Node
+  - `{NODE_ADDRESS}` - IP address of currently checked LB Node
+  - `{ACTIVE_NODES_NAMES}` - hostnames of LB Nodes of currently checked LB Pool
+    considered up by testtool
+  - `{ACTIVE_NODES_ADDRESSES}` - IP addresses of LB Nodes of currently checked
+    LB Pool considered up by testtool
+
 * `hc_ok_codes`: List of HTTP codes treated as "OK"
 * `hc_host`: Host header to be sent
 

bsd_port/Makefile

@@ -1,20 +1,21 @@
 VALID_CATEGORIES+= igports
 PORTNAME=	testtool
-PORTVERSION=	_PV_
+DISTVERSION=	_DV_
 CATEGORIES=	igports
 
 MAINTAINER=	kajetan.staszkiewicz@innogames.de
 COMMENT=	Testtool checks health of loadbalanced services.
 
 USES=		cmake
 
-LIB_DEPENDS=	libfmt.so:devel/libfmt \
-		libyaml-cpp.so:devel/yaml-cpp
+LIB_DEPENDS=	libyaml-cpp.so:devel/yaml-cpp
 
 BUILD_DEPENDS=	${NONEXISTENT}:devel/libevent \
 		${NONEXISTENT}:devel/boost-libs \
 		${NONEXISTENT}:databases/postgresql94-client \
-		${NONEXISTENT}:devel/nlohmann-json
+		${NONEXISTENT}:devel/nlohmann-json \
+		${NONEXISTENT}:devel/googletest \
+		${NONEXISTENT}:devel/libfmt
 
 PLIST_FILES=	/usr/local/sbin/testtool \
 		/usr/local/etc/rc.d/testtool

src/config.cpp

@@ -29,5 +29,5 @@ template bool safe_get(const nlohmann::json &, const char *, bool);
 template int safe_get(const nlohmann::json &, const char *, int);
 template std::string safe_get(const nlohmann::json &, const char *,
                               std::string);
-template std::vector<std::string> safe_get(const nlohmann::json &, const char *,
-                                           std::vector<std::string>);
+template std::vector<int> safe_get(const nlohmann::json &, const char *,
+                                   std::vector<int>);

src/healthcheck.cpp

@@ -4,6 +4,8 @@
 // Copyright (c) 2018 InnoGames GmbH
 //
 
+#define FMT_HEADER_ONLY
+
 #include <fmt/format.h>
 #include <fmt/printf.h>
 #include <iostream>
@@ -15,6 +17,7 @@
 #include "config.h"
 #include "healthcheck.h"
 #include "healthcheck_dns.h"
+#include "healthcheck_dummy.h"
 #include "healthcheck_http.h"
 #include "healthcheck_ping.h"
 #include "healthcheck_postgres.h"
@@ -91,15 +94,15 @@ Healthcheck::Healthcheck(const nlohmann::json &config,
   this->ran = false;
 
   // Initialize healthchecks state basing on state of parent node.
-  // Proper initial state for the healthcheck quarantees no
+  // Proper initial state for the healthcheck guarantees no
   // unnecessary messages.
-  if (parent_lbnode->get_state() == LbNode::STATE_UP) {
-    hard_state = STATE_UP;
-    last_state = STATE_UP;
+  if (parent_lbnode->is_up()) {
+    hard_state = HealthcheckState::STATE_UP;
+    last_state = HealthcheckState::STATE_UP;
     failure_counter = 0;
   } else {
-    hard_state = STATE_DOWN;
-    last_state = STATE_DOWN;
+    hard_state = HealthcheckState::STATE_DOWN;
+    last_state = HealthcheckState::STATE_DOWN;
     failure_counter = max_failed_checks;
   }
 }
@@ -130,10 +133,12 @@ Healthcheck *Healthcheck::healthcheck_factory(const nlohmann::json &config,
         new Healthcheck_postgres(config, _parent_lbnode, ip_address);
   else if (type == "dns")
     new_healthcheck = new Healthcheck_dns(config, _parent_lbnode, ip_address);
+  else if (type == "dummy")
+    new_healthcheck = new Healthcheck_dummy(config, _parent_lbnode, ip_address);
   else
     return NULL;
 
-  log(MSG_INFO, new_healthcheck, "state: created");
+  log(MessageType::MSG_INFO, new_healthcheck, "state: created");
 
   return new_healthcheck;
 }
@@ -158,7 +163,7 @@ int Healthcheck::schedule_healthcheck(struct timespec *now) {
   is_running = true;
 
   if (verbose > 1)
-    log(MSG_INFO, this, "scheduling");
+    log(MessageType::MSG_INFO, this, "scheduling");
 
   return true;
 }
@@ -176,26 +181,33 @@ void Healthcheck::finalize() {
 // the health check.  If it wouldn't be called, the process is not
 // going to continue.
 void Healthcheck::end_check(HealthcheckResult result, string message) {
-  msgType log_type;
+  MessageType log_type;
   string statemsg;
 
   switch (result) {
-  case HC_PASS:
-    log_type = MSG_STATE_UP;
-    this->last_state = STATE_UP;
+  case HealthcheckResult::HC_PASS:
+    log_type = MessageType::MSG_STATE_UP;
+    this->last_state = HealthcheckState::STATE_UP;
     statemsg = fmt::sprintf("state: up message: %s", message);
     this->handle_result(statemsg);
     break;
 
-  case HC_FAIL:
-    log_type = MSG_STATE_DOWN;
-    this->last_state = STATE_DOWN;
+  case HealthcheckResult::HC_FAIL:
+    log_type = MessageType::MSG_STATE_DOWN;
+    this->last_state = HealthcheckState::STATE_DOWN;
     statemsg = fmt::sprintf("state: down message: %s", message);
     this->handle_result(statemsg);
     break;
 
-  case HC_PANIC:
-    log_type = MSG_CRIT;
+  case HealthcheckResult::HC_DRAIN:
+    log_type = MessageType::MSG_STATE_DOWN;
+    this->last_state = HealthcheckState::STATE_DRAIN;
+    statemsg = fmt::sprintf("state: down with draining, message: %s", message);
+    this->handle_result(statemsg);
+    break;
+
+  case HealthcheckResult::HC_PANIC:
+    log_type = MessageType::MSG_CRIT;
     statemsg = fmt::sprintf("state: failure message: %s", message);
     log(log_type, this, statemsg);
     exit(2);
@@ -212,33 +224,44 @@ void Healthcheck::end_check(HealthcheckResult result, string message) {
 void Healthcheck::handle_result(string message) {
   string fail_message;
   bool changed = false;
-  int log_level = MSG_INFO;
+  MessageType log_level = MessageType::MSG_INFO;
 
   // If a healtcheck has passed, zero the failure counter.
-  if (last_state == STATE_UP)
+  if (last_state == HealthcheckState::STATE_UP)
     failure_counter = 0;
 
-  // Change from DOWN to UP. The healthcheck has passed again.
-  if (hard_state == STATE_DOWN && last_state == STATE_UP) {
-    hard_state = STATE_UP;
+  if (hard_state == HealthcheckState::STATE_UP) {
+    switch (last_state) {
+    case HealthcheckState::STATE_UP:
+      // No change, make compiler happy.
+      break;
+    case HealthcheckState::STATE_DOWN:
+      // Change from UP to DOWN. The healthcheck has failed.
+      changed = true;
+      log_level = MessageType::MSG_STATE_DOWN;
+      failure_counter++;
+      fail_message =
+          fmt::sprintf("failure: %d of %d", failure_counter, max_failed_checks);
+      // Mark the hard DOWN state only after the number of failed checks is
+      // reached.
+      if (failure_counter >= max_failed_checks) {
+        hard_state = HealthcheckState::STATE_DOWN;
+      }
+      break;
+    case HealthcheckState::STATE_DRAIN:
+      // Change from UP to DRAIN. The healthcheck has failed with draining.
+      changed = true;
+      log_level = MessageType::MSG_STATE_DOWN;
+      // Make it fail immediately not waiting for max_failed.
+      hard_state = HealthcheckState::STATE_DRAIN;
+      break;
+    }
+  } else if (last_state == HealthcheckState::STATE_UP) {
+    // Change from any state to UP. The healthcheck has passed again.
+    hard_state = HealthcheckState::STATE_UP;
     failure_counter = 0;
     changed = true;
-    log_level = MSG_STATE_UP;
-  }
-  // Change from UP to DOWN. The healthcheck has failed.
-  else if (hard_state == STATE_UP && last_state == STATE_DOWN) {
-    changed = true;
-    log_level = MSG_STATE_DOWN;
-
-    failure_counter++;
-    fail_message =
-        fmt::sprintf("failure: %d of %d", failure_counter, max_failed_checks);
-
-    // Mark the hard DOWN state only after the number of failed checks is
-    // reached.
-    if (failure_counter >= max_failed_checks) {
-      hard_state = STATE_DOWN;
-    }
+    log_level = MessageType::MSG_STATE_UP;
   }
 
   if (changed || verbose) {