Add DKTL_NO_PROXY flag that supresses starting the proxy on CI systems (QA builder version)

README.md

@@ -1,7 +1,12 @@
 # DKAN Tools
 
-This CLI application provides tools for implementing, developing, and maintaining DKAN, the Drupal-based open data catalog. 
+This CLI application provides tools for implementing, developing, and maintaining DKAN, the Drupal-based open data catalog.
 
-For Drupal 7.x projects use the 1.x branch.
+**For Drupal 7.x projects use the 1.x branch.**
 
 [Documentation](https://getdkan.github.io/dkan-tools/)
+
+- `pip install mkdocs`
+- `pip install mkdocs-windmill`
+- `mkdocs serve`
+- Review the docs at http://127.0.0.1:8000

assets/docker/docker-compose.common.yml

@@ -50,7 +50,7 @@ services:
   # Used for all console commands and tools.
   cli:
     hostname: cli
-    image: getdkan/dkan-cli:v0.3.0
+    image: getdkan/dkan-cli:v0.3.1
     env_file:
       - "${DKTL_DIRECTORY}/assets/docker/mysql.env"
     environment:

bin/dktl

@@ -112,11 +112,11 @@ set_project_directory() {
     echo "$path"
   }
 
-  DKTL_PROJECT_DIRECTORY=$(find_up dktl.yml)
-  if [ -z "$DKTL_PROJECT_DIRECTORY" ]; then
-    if [ "$1" = "init" ] || [ "$1" = "down" ]; then
-      DKTL_PROJECT_DIRECTORY=$(pwd)
-    else
+  if [ "$1" = "init" ] || [ "$1" = "down" ]; then
+    DKTL_PROJECT_DIRECTORY=$(pwd)
+  else
+    DKTL_PROJECT_DIRECTORY=$(find_up dktl.yml)
+    if [ -z "$DKTL_PROJECT_DIRECTORY" ]; then
       echo "DKTL is running outside of a DKTL project. Run dktl init in the project directory first."
       exit 1
     fi
@@ -193,9 +193,11 @@ dc_base() {
 
 # Set up subdomain with traefic proxy.
 proxy_connect () {
-  network="${DKTL_SLUG}_default"
-  if docker network connect "$network" proxy >/dev/null 2>&1; then
-    echo "Connected dktl-proxy to \"${network}\" network."
+  if [ -z "$DKTL_NO_PROXY" ]; then
+    network="${DKTL_SLUG}_default"
+    if docker network connect "$network" proxy >/dev/null 2>&1; then
+      echo "Connected dktl-proxy to \"${network}\" network."
+    fi
   fi
 }
 
@@ -213,32 +215,35 @@ dc_up () {
 dc_down() {
   network="${DKTL_SLUG}_default"
 
-  if docker network disconnect "$network" dktl-proxy >/dev/null 2>&1; then
-    echo "Disconnected dktl-proxy from \"${network}\" network."
+  if [ -z "$DKTL_NO_PROXY" ]; then
+    if docker network disconnect "$network" dktl-proxy >/dev/null 2>&1; then
+      echo "Disconnected dktl-proxy from \"${network}\" network."
+    fi
   fi
 
   dc_base down -v "${@:2}"
 }
 
 # Make sure the proxy container is up.
 proxy_setup() {
-  proxy_service_ps=$(docker ps -f 'name=dktl-proxy' --format '{{.Names}}' 2>/dev/null)
-  if [[ -z $proxy_service_ps ]]; then
-    echo "Running dktl-proxy.."
-    # Make sure no old instance is using the name.
-    docker rm dktl-proxy >/dev/null 2>&1
-    if ! docker run -d -p 8080:8080 -p 80:80 -p 433:433 \
-      -v '/var/run/docker.sock:/var/run/docker.sock' \
-      --name 'dktl-proxy' \
-      traefik:v2.0 \
-      --log.level=INFO --api.insecure=true --providers.docker=true \
-      --providers.docker.exposedbydefault=false --entrypoints.web.address=:80 \
-      --entrypoints.websecure.address=:443 \
-      >/dev/null; \
-    then
-      echo -n "Failed to start the dktl-proxy container..."
+  if [ -z "$DKTL_NO_PROXY" ]; then
+    proxy_service_ps=$(docker ps -f 'name=dktl-proxy' --format '{{.Names}}' 2>/dev/null)
+    if [[ -z $proxy_service_ps ]]; then
+      echo "Running dktl-proxy.."
+      # Make sure no old instance is using the name.
+      docker rm dktl-proxy >/dev/null 2>&1
+      if ! docker run -d -p 8080:8080 -p 80:80 -p 433:433 \
+        -v '/var/run/docker.sock:/var/run/docker.sock' \
+        --name 'dktl-proxy' \
+        traefik:v2.0 \
+        --log.level=INFO --api.insecure=true --providers.docker=true \
+        --providers.docker.exposedbydefault=false --entrypoints.web.address=:80 \
+        --entrypoints.websecure.address=:443 \
+        >/dev/null; \
+      then
+        echo -n "Failed to start the dktl-proxy container..."
+      fi
     fi
-
   fi
 }
 

docs/docker_config.md

@@ -0,0 +1,19 @@
+# Modifying the Docker containers
+
+By default, dkan-tools will run docker containers with everything needed for a DKAN site to run.
+
+Sometimes we need to modify those docker containers.
+
+A common example of these kinds of modification is adding an environment variable to the web, or cli containers.
+
+dkan-tools provides a mechanism to do just this.
+
+The configuration that dkan-tools uses to start the docker containers is in `assets/docker` in the dkan-tools repo.
+
+To make changes to that configuration, you can add a file named `docker-compose.overrides.yml` to the `src/docker` directory **in your project**.
+
+Any valid docker-compose configuration can be added to that file and it will be merged with the default configuration from `assets/docker/docker-compose.common.yml` before the containers are created.
+
+!!! note "Seeing your configuration"
+
+    After adding your file and making configuration changes you will have to remove and recreate your containers, you can do this by running `dktl dc kill && dktl dc rm && dktl`

docs/index.md

@@ -12,7 +12,6 @@ DKAN Tools was designed with a [Docker](https://www.docker.com/)-based local dev
 
 That's it! All other dependencies are included in the Docker containers that dkan-tools will create.
 
-
 ## Basic usage
 
 Once you are working in an initialized project folder, you can type `dktl` at any time to see a list of all available commands.

mkdocs.yml

@@ -16,6 +16,7 @@ nav:
     - 'Site Maintenance': 'maintenance.md'
     - 'patching.md'
     - 'Custom commands': 'extend.md'
+    - 'Modifying the Docker containers': 'docker_config.md'
     - 'troubleshooting.md'
 markdown_extensions:
     - admonition

src/Command/InitCommands.php

@@ -67,7 +67,7 @@ private function initSrc()
             return;
         }
 
-        $directories = ['docker', 'modules', 'themes', 'site', 'test', 'script', 'command'];
+        $directories = ['docker', 'modules', 'themes', 'libraries', 'site', 'test', 'script', 'command'];
         foreach ($directories as $directory) {
             $dir = "src/{$directory}";
             $result = $this->_mkdir($dir);

src/Command/MakeCommands.php

@@ -40,14 +40,14 @@ public function make($opts = [
         'prefer-source' => false,
         'prefer-dist' => true,
         'optimize-autoloader' => false,
-        'dev' => true,
+        'no-dev' => false,
     ])
     {
         $this->io()->section("Running dktl make");
 
         // Run composer install while passing the options.
         $composerInstall = $this->taskComposerInstall();
-        $composerOptions = ['prefer-source', 'prefer-dist', 'optimize-autoloader', 'dev'];
+        $composerOptions = ['prefer-source', 'prefer-dist', 'optimize-autoloader', 'no-dev'];
         foreach ($composerOptions as $opt) {
             if ($opts[$opt]) {
                 $composerInstall->option($opt);
@@ -77,6 +77,7 @@ public function makeSymlinks()
             ['target' => 'src/site',    'link' => '/sites/default'],
             ['target' => 'src/modules', 'link' => '/modules/custom'],
             ['target' => 'src/themes',  'link' => '/themes/custom'],
+            ['target' => 'src/libraries',  'link' => '/libraries'],
             ['target' => 'src/schema',  'link' => '/schema'],
         ];
         foreach ($targetsAndLinks as $targetAndLink) {

tests/dktl_test.sh

@@ -47,6 +47,7 @@ testDktlInit() {
     result=`ls src`
     assertContains "${result}" "command"
     assertContains "${result}" "docker"
+    assertContains "${result}" "libraries"
     assertContains "${result}" "modules"
     assertContains "${result}" "script"
     assertContains "${result}" "site"
@@ -105,7 +106,18 @@ testBringDown() {
     assertNotContains "${result}" "${DKTL_TEST_PROJECT_NAME}_default"
 }
 
+testDktlInitProxy() {
+    dktl proxy:kill
+    result=`dktl url`
+    assertContains "${result}" "Running dktl-proxy.."
+    dktl proxy:kill
+    result=`DKTL_NO_PROXY=1 dktl url`
+    assertNotContains "${result}" "proxy"
+}
+
+
 oneTimeTearDown() {
+    cd "${DKTL_TEST_DIR}"
     containers=`docker ps --filter name="${DKTL_TEST_PROJECT_NAME}*" -aq`
     echo $containers
     if [ ! -z "$containers" ]; then
@@ -119,4 +131,4 @@ oneTimeTearDown() {
 }
 
 # Load shUnit2.
-. ./shunit2/shunit2
+. ./shunit2/shunit2