Merge pull request #731 from srvrco/acme-dns

Fix CNAME issues and support acme-dns (fixes #722, #308, #646, #600, #585)

Author: timkimber

.github/workflows/run-tests-pebble.yml

@@ -92,6 +92,14 @@ jobs:
         run: docker-compose up -d --build
       - name: Run test suite on Ubuntu
         run: test/run-test.sh ubuntu
+  test-ubuntu14:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu14
+        run: test/run-test.sh ubuntu14
   test-ubuntu16:
     runs-on: ubuntu-latest
     steps:

.github/workflows/run-tests-staging-acmedns.yml

@@ -0,0 +1,25 @@
+name: Run all tests using Dynu and acmedns
+on:
+  push:
+    paths-ignore:
+      - '.github/workflows/*'
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch:
+    branches:
+      - master
+env:
+  DYNU_API_KEY: ${{ secrets.DYNU_API_KEY == '' && '65cXefd35XbYf36546eg5dYcZT6X52Y2' || secrets.DYNU_API_KEY }}
+jobs:
+  test-ubuntu-acmedns:
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu against Staging using acmedns
+        run: test/run-test.sh ubuntu-acmedns

.gitignore

@@ -7,3 +7,4 @@
 *.tar.gz
 *.orig
 JSON.sh
+.vscode/settings.json

dns_scripts/dns_add_acmedns

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Need to add your API user and key below or set as env variable
+apiuser=${ACMEDNS_API_USER:-''}
+apikey=${ACMEDNS_API_KEY:-''}
+apisubdomain=${ACMEDNS_SUBDOMAIN:-''}
+
+# This script adds a token to acme-dns.io DNS for the ACME challenge
+# usage dns_add_acme-dns "domain name" "token"
+# return codes are;
+# 0 - success
+# 1 - error returned from server
+
+fulldomain="${1}"
+token="${2}"
+
+API='https://auth.acme-dns.io/update'
+
+# Check initial parameters
+if [[ -z "$fulldomain" ]]; then
+  echo "DNS script requires full domain name as first parameter"
+  exit 1
+fi
+if [[ -z "$token" ]]; then
+  echo "DNS script requires challenge token as second parameter"
+  exit 1
+fi
+
+curl_params=(
+  -H "accept: application/json"
+  -H "X-Api-Key: $apikey"
+  -H "X-Api-User: $apiuser"
+  -H 'Content-Type: application/json'
+)
+
+generate_post_data()
+{
+  cat <<EOF
+{
+  "subdomain": "$apisubdomain",
+  "txt": "$token"
+}
+EOF
+}
+
+resp=$(curl --silent \
+  "${curl_params[@]}" \
+  -X POST "${API}" \
+  --data "$(generate_post_data)")
+
+echo $resp
+# If adding record failed (returned json includes "error" then print error message
+if [[ "$resp" = *"\"error\""* ]]; then
+  echo "Error: DNS challenge not added: unknown error - ${resp}"
+  exit 1
+fi

dns_scripts/dns_add_cpanel

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Need to add your API user and key below or set as env variable
+apiuser=${ACMEDNS_API_USER:-''}
+apikey=${ACMEDNS_API_KEY:-''}
+apisubdomain=${ACMEDNS_SUBDOMAIN:-''}
+
+# This script adds a token to acme-dns.io DNS for the ACME challenge
+# usage dns_add_acme-dns "domain name" "token"
+# return codes are;
+# 0 - success
+# 1 - error returned from server
+
+fulldomain="${1}"
+token="${2}"
+
+API='https://auth.acme-dns.io/update'
+
+# Check initial parameters
+if [[ -z "$fulldomain" ]]; then
+  echo "DNS script requires full domain name as first parameter"
+  exit 1
+fi
+if [[ -z "$token" ]]; then
+  echo "DNS script requires challenge token as second parameter"
+  exit 1
+fi
+
+curl_params=(
+  -H "accept: application/json"
+  -H "X-Api-Key: $apikey"
+  -H "X-Api-User: $apiuser"
+  -H 'Content-Type: application/json'
+)
+
+generate_post_data()
+{
+  cat <<EOF
+{
+  "subdomain": "$apisubdomain",
+  "txt": "$token"
+}
+EOF
+}
+
+resp=$(curl --silent \
+  "${curl_params[@]}" \
+  -X POST "${API}" \
+  --data "$(generate_post_data)")
+
+echo $resp
+# If adding record failed (returned json includes "error" then print error message
+if [[ "$resp" = *"\"error\""* ]]; then
+  echo "Error: DNS challenge not added: unknown error - ${resp}"
+  exit 1
+fi

dns_scripts/dns_add_del_aliyun.sh

@@ -276,6 +276,9 @@
 # 2021-10-01 Show help if no domain specified (#705)(2.44)
 # 2021-10-08 Extract release tag from release api using awk (fix BSD issues)
 # 2021-10-11 Fix broken upgrade url (#718)(2.45)
+# 2021-10-22 Copy fullchain to DOMAIN_CHAIN_LOCATION (amartin-git)
+# 2021-11-10 Detect Solaris and use gnu tools (#701)(miesi)
+# 2021-11-12 Support acme-dns and fix CNAME issues (#722)(#308)
 # ----------------------------------------------------------------------------------------
 
 case :$SHELLOPTS: in
@@ -536,12 +539,6 @@ check_challenge_completion() { # checks with the ACME server if our challenge is
     debug "sleep 5 secs before testing verify again"
     sleep 5
   done
-
-  if [[ "$DEACTIVATE_AUTH" == "true" ]]; then
-    deactivate_url=$(echo "$responseHeaders" | grep "^Link" | awk -F"[<>]" '{print $2}')
-    deactivate_url_list="$deactivate_url_list $deactivate_url"
-    debug "adding url to deactivate list - $deactivate_url"
-  fi
 }
 
 check_challenge_completion_dns() { # perform validation via DNS challenge
@@ -575,10 +572,19 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
         # shellcheck disable=SC2086
         debug "$DNS_CHECK_FUNC" $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}"
         # shellcheck disable=SC2086
-        check_result=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}" \
-                      | grep -i "^${rr}" \
-                      | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
+        check_output=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}")
+        check_result=$(grep -i "^${rr}"<<<"${check_output}"|grep 'IN\WTXT'|awk -F'"' '{ print $2}')
         debug "check_result=\"$check_result\""
+
+        # Check if rr is a CNAME
+        if [[ -z "$check_result" ]]; then
+          rr_cname=$(grep -i "^${rr}"<<<"${check_output}"|grep 'IN\WCNAME'|awk '{ print $5}')
+          debug "cname check=\"$rr_cname\""
+          if [[ -n "$rr_cname" ]]; then
+            check_result=$(grep -i "^${rr_cname}"<<<"${check_output}"|grep 'IN\WTXT'|awk -F'"' '{ print $2}' | uniq)
+          fi
+        fi
+
         if [[ -z "$check_result" ]]; then
           # shellcheck disable=SC2086
           debug "$DNS_CHECK_FUNC" $DNS_CHECK_OPTIONS ANY "${rr}" "@${ns}"
@@ -589,14 +595,20 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
           debug "check_result=\"$check_result\""
         fi
       elif [[ "$DNS_CHECK_FUNC" == "host" ]]; then
+        debug "$DNS_CHECK_FUNC" -t TXT "${rr}" "${ns}"
         check_result=$($DNS_CHECK_FUNC -t TXT "${rr}" "${ns}" \
                       | grep 'descriptive text'|awk -F'"' '{ print $2}')
+        debug "check_result=\"$check_result\""
       else
+        debug "$DNS_CHECK_FUNC" -type=txt "${rr}" "${ns}"
         check_result=$(nslookup -type=txt "${rr}" "${ns}" \
                       | grep 'text ='|awk -F'"' '{ print $2}')
+        debug "check_result=\"$check_result\""
         if [[ -z "$check_result" ]]; then
+          debug "$DNS_CHECK_FUNC" -type=any "${rr}" "${ns}"
           check_result=$(nslookup -type=any "${rr}" "${ns}" \
                       | grep 'text ='|awk -F'"' '{ print $2}')
+          debug "check_result=\"$check_result\""
         fi
       fi
       debug "expecting  \"$auth_key\""
@@ -1201,6 +1213,11 @@ create_order() {
         fi
         ((dn++))
       done
+      if [[ "$DEACTIVATE_AUTH" == "true" ]]; then
+        deactivate_url_list+=" $l "
+        debug "url added to deactivate list ${l}"
+        debug "deactivate list is now $deactivate_url_list"
+      fi
     done
   fi
 }
@@ -1352,12 +1369,6 @@ for d in "${alldomains[@]}"; do
 
   if [[ $response_status == "valid" ]]; then
     info "$d is already validated"
-    if [[ "$DEACTIVATE_AUTH" == "true" ]]; then
-      deactivate_url="$(echo "$responseHeaders" | awk ' $1 ~ "^Location" {print $2}' | tr -d "\r")"
-      deactivate_url_list+=" $deactivate_url "
-      debug "url added to deactivate list ${deactivate_url}"
-      debug "deactivate list is now $deactivate_url_list"
-    fi
     # increment domain-counter
     ((dn++))
   else
@@ -1554,6 +1565,7 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
       # domain is a CNAME: resolve it and continue with that
       debug Domain is a CNAME, actual domain is "$cname"
       gad_d=${cname}
+      res=
     fi
 
     # Use SOA +trace to find the name server
@@ -1629,6 +1641,7 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
         primary_ns="$primary_ns $PUBLIC_DNS_SERVER"
       fi
 
+      debug set primary_ns="$primary_ns"
       return
     fi
   fi
@@ -1639,26 +1652,48 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
     # shellcheck disable=SC2086
     res=$(nslookup $DNS_CHECK_OPTIONS -debug -type=soa -type=ns "$gad_d" ${gad_s})
 
+    # check for CNAME (assumes gad_d is _acme-challenge.{host})
+    if [[ "$(grep -c "NXDOMAIN"<<<"$res")" -gt 0 ]]; then
+      debug "Cannot find nameserver record for $gad_d, using parent domain ${gad_d#*.}"
+      gad_d="${gad_d#*.}"
+      debug "nslookup $DNS_CHECK_OPTIONS -debug -type=soa -type=ns $gad_d ${gad_s}"
+      # shellcheck disable=SC2086
+      res=$(nslookup $DNS_CHECK_OPTIONS -debug -type=soa -type=ns "$gad_d" ${gad_s})
+    fi
+
     if [[ "$(echo "$res" | grep -c "Non-authoritative")" -gt 0 ]]; then
       # this is a Non-authoritative server, need to check for an authoritative one.
+      debug "Response from non-authoritative server looking for authoritative server"
+
       gad_s=$(echo "$res" | awk '$2 ~ "nameserver" {print $4; exit }' |sed 's/\.$//g')
-      if [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
+      # If the previous line fails to find the nameserver, use the original
+      if [[ -z "$gad_s" ]]; then
+        gad_s="$orig_gad_s"
+      fi
+
+      if [[ "$(echo "$res" | grep -c "canonical name")" -gt 0 ]]; then
+        debug "$gad_d" appears to be a CNAME
+        gad_d=$(echo "$res" | awk ' $2 ~ "canonical" {print $5; exit }' |sed 's/\.$//g')
+        debug "Using $gad_d instead"
+      elif [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
         # if domain name doesn't exist, then find auth servers for next level up
+        debug "Couldn't find NS or SOA for domain name, using nslookup $DNS_CHECK_OPTIONS -debug ${gad_d#*.} ${orig_gad_s}"
+        # shellcheck disable=SC2086
+        res=$(nslookup $DNS_CHECK_OPTIONS -debug "${gad_d#*.}" ${orig_gad_s})
         gad_s=$(echo "$res" | awk '$1 ~ "origin" {print $3; exit }')
         gad_d=$(echo "$res" | awk '$1 ~ "->" {print $2; exit}')
         # handle scenario where awk returns nothing
         if [[ -z "$gad_d" ]]; then
-          gad_d="$orig_gad_d"
+          gad_d="${orig_gad_d}"
         fi
       fi
 
+      debug "Using nslookup $DNS_CHECK_OPTIONS -debug -type=soa -type=ns $gad_d ${gad_s}"
       # shellcheck disable=SC2086
       res=$(nslookup $DNS_CHECK_OPTIONS -debug -type=soa -type=ns "$gad_d" ${gad_s})
     fi
 
-    if [[ "$(echo "$res" | grep -c "canonical name")" -gt 0 ]]; then
-      gad_d=$(echo "$res" | awk ' $2 ~ "canonical" {print $5; exit }' |sed 's/\.$//g')
-    elif [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
+    if [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
       gad_s=$(echo "$res" | awk ' $1 ~ "origin" {print $3; exit }')
       gad_d=$(echo "$res"| awk '$1 ~ "->" {print $2; exit}')
       # handle scenario where awk returns nothing
@@ -1680,6 +1715,11 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
         primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
       fi
 
+      if [[ "$CHECK_PUBLIC_DNS_SERVER" == "true" ]]; then
+        primary_ns="$primary_ns $PUBLIC_DNS_SERVER"
+      fi
+
+      debug set primary_ns="$primary_ns"
       return
     fi
   fi

dns_scripts/dns_del_acmedns

@@ -14,6 +14,9 @@ setup() {
     if [ -f /usr/bin/dig ]; then
         mv /usr/bin/dig /usr/bin/dig.getssl.bak
     fi
+    if [ -f /usr/bin/drill ]; then
+        mv /usr/bin/drill /usr/bin/drill.getssl.bak
+    fi
     if [ -f /usr/bin/host ]; then
         mv /usr/bin/host /usr/bin/host.getssl.bak
     fi
@@ -25,6 +28,9 @@ teardown() {
     if [ -f /usr/bin/dig.getssl.bak ]; then
         mv /usr/bin/dig.getssl.bak /usr/bin/dig
     fi
+    if [ -f /usr/bin/drill.getssl.bak ]; then
+        mv /usr/bin/drill.getssl.bak /usr/bin/drill
+    fi
     if [ -f /usr/bin/host.getssl.bak ]; then
         mv /usr/bin/host.getssl.bak /usr/bin/host
     fi

dns_scripts/dns_del_cpanel

@@ -38,6 +38,10 @@ check_github_quota() {
 
 
 setup_file() {
+    if [ -n "$STAGING" ]; then
+        echo "Using staging server, skipping internal test" >&3
+        return 0
+    fi
     if [ -f $BATS_RUN_TMPDIR/failed.skip ]; then
         echo "# Skipping setup due to previous test failure" >&3
         return 0