Get Livebook running alongside Meadow in staging

Start prepping for livebook deploy
Use new Livebook custom auth
Add livebook to task definition & load balancer
Add SuperUser role (for Livebook access)
Add internal DNS service entry so Livebook & Meadow can talk to each other
Fix stream-authorizer Terraform so it doesn't constantly think it has to redeploy

.github/scripts/configure_aws.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
-environment=$(echo $DEPLOY_ENV | tr a-z A-Z)
+environment=STAGING
+if [[ $DEPLOY_ENV == "production" ]]; then
+  environment=PRODUCTION
+fi
 access_key_id_var=${environment}_AWS_ACCESS_KEY_ID
 access_key_id=$(jq -r ".${access_key_id_var}" <<< $SECRETS)
 secret_key_var=${environment}_AWS_SECRET_ACCESS_KEY

.github/workflows/deploy.yml

@@ -47,12 +47,19 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
+      - uses: docker/build-push-action@v2
+        with:
+          context: ./livebook
+          push: true
+          tags: ${{ steps.login-ecr.outputs.registry }}/meadow:livebook-${{ env.DEPLOY_ENV }}
       - uses: docker/build-push-action@v2
         with:
           context: ./app
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/meadow:${{ env.DEPLOY_ENV }}
           build-args: |
+            BUILD_IMAGE=hexpm/elixir:1.15.7-erlang-26.0.2-debian-bullseye-20231009
+            RUNTIME_IMAGE=node:18-bullseye-slim
             HONEYBADGER_API_KEY=${{ secrets.HONEYBADGER_API_KEY }}
             HONEYBADGER_API_KEY_FRONTEND=${{ secrets.HONEYBADGER_API_KEY_FRONTEND }}
             HONEYBADGER_ENVIRONMENT=${{ env.DEPLOY_ENV }}

.gitignore

@@ -70,4 +70,4 @@ yarn.lock
 .DS_Store
 **/*/.DS_Store
 
-lambdas/stream-authorizer/environment.json
+lambdas/stream-authorizer/config

app/Dockerfile

@@ -1,5 +1,8 @@
+ARG BUILD_IMAGE
+ARG RUNTIME_IMAGE
+
 # Install elixir & npm dependencies
-FROM hexpm/elixir:1.15.4-erlang-26.0.2-alpine-3.18.2 AS build
+FROM ${BUILD_IMAGE} AS build
 LABEL edu.northwestern.library.app=meadow \
   edu.northwestern.library.cache=true \
   edu.northwestern.library.stage=deps
@@ -9,17 +12,19 @@ ARG HONEYBADGER_ENVIRONMENT=
 ARG HONEYBADGER_REVISION=
 ARG MEADOW_VERSION=
 ENV MIX_ENV=prod
-RUN  apk add --update --repository https://dl-3.alpinelinux.org/alpine/edge/testing/ curl git libstdc++ \
-  && mix local.hex --force \
+RUN  mix local.hex --force \
   && mix local.rebar --force
-ENV NODE_VERSION 18.18.0
+ENV NODE_VERSION 18
 ENV NPM_VERSION 10.1.0
 ENV ARCH x64
-RUN curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz"; \
-  tar -xJf "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
-  && ln -s /usr/local/bin/node /usr/local/bin/nodejs && \
-  npm install -g npm@$NPM_VERSION; \
-  rm -f "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz";
+RUN apt update -qq \
+ && apt install -y ca-certificates curl git gnupg \
+ && mkdir -p /etc/apt/keyrings \
+ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
+ && apt update -qq \
+ && apt install -y nodejs \
+ && npm install -g npm@$NPM_VERSION
 COPY . /app
 WORKDIR /app
 RUN mix deps.get --only prod \
@@ -39,12 +44,12 @@ WORKDIR /app
 RUN mix release --overwrite
 
 # Create runtime image
-FROM node:18-alpine
+FROM ${RUNTIME_IMAGE}
 LABEL edu.northwestern.library.app=meadow \
   edu.northwestern.library.stage=runtime
-RUN apk update && apk --no-cache --update add curl jq libcrypto3 ncurses-libs openssl-dev
+RUN apt update -qq && apt install -y curl jq libssl-dev libncurses5-dev
 ENV LANG=en_US.UTF-8
-EXPOSE 4000 4369 24601
+EXPOSE 4000 4369
 COPY --from=build /app/_build/prod/rel/meadow /app
 WORKDIR /app
 ENTRYPOINT ["./bin/meadow"]

app/assets/js/components/Auth/DisplayAuthorized.jsx

@@ -10,7 +10,7 @@ function AuthDisplayAuthorized({ level, children }) {
 }
 
 AuthDisplayAuthorized.propTypes = {
-  level: PropTypes.oneOf(["USER", "EDITOR", "MANAGER", "ADMINISTRATOR"]),
+  level: PropTypes.oneOf(["USER", "EDITOR", "MANAGER", "ADMINISTRATOR", "SUPERUSER"]),
   children: PropTypes.node,
 };
 

app/assets/js/components/Livebook/Livebook.jsx

@@ -0,0 +1,23 @@
+import React from "react";
+import { useQuery } from "@apollo/client";
+import { LIVEBOOK_URL } from "@js/components/UI/ui.gql";
+
+export default function LivebookLink({children}) {
+  const { data, loading, error } = useQuery(LIVEBOOK_URL);
+
+  if (error) {
+    return (
+      <p className="notifcation is-danger">
+        There was an error retrieving the Livebook url
+      </p>
+    );
+  }
+
+  if (loading) {
+    return null;
+  }
+
+  return (
+    data?.livebookUrl?.url ? <a href={data.livebookUrl.url} target="_blank">{children}</a> : <></>
+  )
+}

app/assets/js/components/UI/Layout/NavBar.jsx

@@ -4,6 +4,7 @@ import { useLocation } from "react-router-dom";
 import { AuthContext } from "../../Auth/Auth";
 import client from "../../../client";
 import UISearchBar from "../SearchBar";
+import UILivebookLink from "@js/components/Livebook/Livebook";
 import UILayoutNavDropdown from "@js/components/UI/Layout/NavDropdown";
 import UILayoutNavDropdownHeader from "@js/components/UI/Layout/NavDropdownHeader";
 import UILayoutNavDropdownBody from "@js/components/UI/Layout/NavDropdownBody";
@@ -157,6 +158,15 @@ const UILayoutNavBar = () => {
                     <UILayoutNavDropdownBody
                       isExpanded={activeHoverNav === "Dashboards"}
                     >
+                      <AuthDisplayAuthorized level="SUPERUSER">
+                        <UILayoutNavDropdownItem>
+                          <UILivebookLink>
+                            <IconText icon={<GrMultiple />}>
+                              Livebook
+                            </IconText>
+                          </UILivebookLink>
+                        </UILayoutNavDropdownItem>
+                      </AuthDisplayAuthorized>
                       <UILayoutNavDropdownItem>
                         <Link to="/dashboards/batch-edit">
                           <IconText icon={<GrMultiple />}>

app/assets/js/components/UI/ui.gql.js

@@ -15,3 +15,11 @@ export const GET_DCAPI_ENDPOINT = gql`
     }
   }
 `;
+
+export const LIVEBOOK_URL = gql`
+  query LivebookUrl {
+    livebookUrl {
+      url
+    }
+  }
+`

app/assets/js/hooks/useIsAuthorized.js

@@ -10,7 +10,7 @@ import { Notification } from "@nulib/design-system";
  */
 
 // Order of role-based access low to high
-const userRoleHierarchy = ["USER", "EDITOR", "MANAGER", "ADMINISTRATOR"];
+const userRoleHierarchy = ["USER", "EDITOR", "MANAGER", "ADMINISTRATOR", "SUPERUSER"];
 
 export default function useIsAuthorized() {
   const { data, loading, error } = useQuery(GET_CURRENT_USER_QUERY);

app/config/releases.exs

@@ -116,6 +116,8 @@ config :meadow,
   preservation_check_bucket: aws_secret("meadow", dig: ["buckets", "preservation_check"]),
   streaming_bucket: aws_secret("meadow", dig: ["buckets", "streaming"])
 
+config :meadow, :livebook, url: environment_secret("LIVEBOOK_URL", default: nil)
+
 config :logger, level: :info
 
 config :meadow, Meadow.Scheduler,
@@ -178,7 +180,10 @@ config :hackney,
               processors: [
                 default: [
                   concurrency:
-                    environment_secret("#{key}_PROCESSOR_CONCURRENCY", cast: :integer, default: 10),
+                    environment_secret("#{key}_PROCESSOR_CONCURRENCY",
+                      cast: :integer,
+                      default: 10
+                    ),
                   max_demand:
                     environment_secret("#{key}_MAX_DEMAND", cast: :integer, default: 10),
                   min_demand: environment_secret("#{key}_MIN_DEMAND", cast: :integer, default: 5)

app/lib/meadow/constants.ex

@@ -5,7 +5,7 @@ defmodule Meadow.Constants do
     quote do
       @ingest_sheet_headers ~w(description file_accession_number filename label
         role structure work_accession_number work_image work_type)
-      @role_priority ~w[Administrators Managers Editors Users]
+      @role_priority ~w[SuperUsers Administrators Managers Editors Users]
     end
   end
 end

app/lib/meadow/roles.ex

@@ -11,6 +11,12 @@ defmodule Meadow.Roles do
     iex> authorized?("User", :any)
     true
 
+    iex> authorized?("SuperUser", "SuperUser")
+    true
+
+    iex> authorized?("Administrator", "SuperUser")
+    false
+
     iex> authorized?("Administrator", "User")
     true
 
@@ -30,6 +36,8 @@ defmodule Meadow.Roles do
 
   def authorized?(nil, _), do: false
   def authorized?(_, :any), do: true
+  def authorized?("SuperUser", _role), do: true
+  def authorized?("Administrator", "SuperUser"), do: false
   def authorized?("Administrator", _role), do: true
   def authorized?("Manager", "Editor"), do: true
   def authorized?("Manager", "User"), do: true

app/lib/meadow_web/resolvers/helpers.ex

@@ -42,6 +42,10 @@ defmodule MeadowWeb.Resolvers.Helpers do
     {:ok, %{url: Config.iiif_server_url()}}
   end
 
+  def get_livebook_url(_, _, _) do
+    {:ok, %{url: Application.get_env(:meadow, :livebook, []) |> Keyword.get(:url)}}
+  end
+
   def dcapi_endpoint(_, _args, _) do
     {:ok, %{url: Application.get_env(:meadow, :dc_api) |> get_in([:v2, "base_url"])}}
   end

app/lib/meadow_web/schema/schema.ex

@@ -67,6 +67,10 @@ defmodule MeadowWeb.Schema do
     field :expires, non_null(:datetime)
   end
 
+  object :nullable_url do
+    field :url, :string
+  end
+
   object :url do
     field :url, non_null(:string)
   end

app/lib/meadow_web/schema/types/account_types.ex

@@ -73,6 +73,7 @@ defmodule MeadowWeb.Schema.AccountTypes do
 
   @desc "Meadow user roles"
   enum :user_role do
+    value(:superuser, as: "SuperUser", description: "superuser")
     value(:administrator, as: "Administrator", description: "administrator")
     value(:manager, as: "Manager", description: "manager")
     value(:editor, as: "Editor", description: "editor")

app/lib/meadow_web/schema/types/helper_types.ex

@@ -45,6 +45,12 @@ defmodule MeadowWeb.Schema.HelperTypes do
       middleware(Middleware.Authenticate)
       resolve(&Resolvers.Helpers.work_archiver_endpoint/3)
     end
+
+    @desc "Get the livebook URL"
+    field :livebook_url, :nullable_url do
+      middleware(Middleware.Authenticate)
+      resolve(&Resolvers.Helpers.get_livebook_url/3)
+    end
   end
 
   enum :s3_upload_type do

app/priv/graphql/schema.json

@@ -7298,6 +7298,18 @@
                 }
               }
             },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": "Get the livebook URL",
+              "isDeprecated": false,
+              "name": "livebookUrl",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Url",
+                "ofType": null
+              }
+            },
             {
               "args": [],
               "deprecationReason": null,
@@ -8313,6 +8325,12 @@
         {
           "description": "Meadow user roles",
           "enumValues": [
+            {
+              "deprecationReason": null,
+              "description": "superuser",
+              "isDeprecated": false,
+              "name": "SUPERUSER"
+            },
             {
               "deprecationReason": null,
               "description": "administrator",

app/rel/env.sh.eex

@@ -6,7 +6,7 @@ if [ "$ECS_CONTAINER_METADATA_URI" != "" ]; then
 fi
 
 # Set the release to work across nodes
-export RELEASE_DISTRIBUTION=name
-export RELEASE_NODE=<%= @release.name %>@${IP_ADDR}
+export RELEASE_DISTRIBUTION=${RELEASE_DISTRIBUTION:-name}
+export RELEASE_NODE=${RELEASE_NODE:-<%= @release.name %>@${IP_ADDR}}
 
 echo "Erlang node name: ${RELEASE_NODE}"

infrastructure/deploy/ecs.tf

@@ -105,12 +105,20 @@ resource "aws_security_group" "meadow_load_balancer" {
   }
 
   ingress {
-    description = "HTTPS in"
+    description = "HTTPS in (Meadow)"
     from_port   = 443
     to_port     = 443
     protocol    = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
   }
+
+  ingress {
+    description = "HTTPS in (Livebook)"
+    from_port   = 8080
+    to_port     = 8080
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
 }
 
 data "aws_iam_policy" "ecs_exec_command" {
@@ -171,6 +179,27 @@ resource "aws_lb_target_group" "meadow_target" {
   }
 }
 
+resource "aws_lb_target_group" "meadow_livebook_target" {
+  port                 = 8080
+  deregistration_delay = 30
+  target_type          = "ip"
+  protocol             = "HTTP"
+  vpc_id               = data.aws_vpc.this_vpc.id
+  tags                 = var.tags
+
+  health_check {
+    path                = "/"
+    matcher             = "200,403"
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+  }
+
+  stickiness {
+    enabled = false
+    type    = "lb_cookie"
+  }
+}
+
 resource "aws_lb" "meadow_load_balancer" {
   name               = "${var.stack_name}-lb"
   internal           = false
@@ -210,6 +239,19 @@ resource "aws_lb_listener" "meadow_lb_listener_https" {
   }
 }
 
+resource "aws_lb_listener" "meadow_lb_listener_livebook" {
+  load_balancer_arn = aws_lb.meadow_load_balancer.arn
+  port              = 8080
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-2016-08"
+  certificate_arn   = data.aws_acm_certificate.meadow_cert.arn
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.meadow_livebook_target.arn
+  }
+}
+
 resource "random_string" "secret_key_base" {
   length  = "64"
   special = "false"