diff --git a/xloader/Dockerfile b/xloader/Dockerfile new file mode 100644 index 0000000..68aa58d --- /dev/null +++ b/xloader/Dockerfile @@ -0,0 +1,91 @@ +# use the official Python 3.10 (bookworm slim) image as the base image +FROM python:3.10-slim-bookworm + +# Tag passed through via the Makefile +ARG CKAN_TAG=${CKAN_TAG} +ARG XLOADER_VERSION=${XLOADER_VERSION} + +ENV TZ=UTC +ENV APP_DIR=/srv/app +ENV SRC_DIR=${APP_DIR}/src +ENV CKAN_INI=${APP_DIR}/ckan.ini +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_STORAGE_PATH=/var/lib/ckan +ENV GIT_URL=https://github.com/ckan/ckan.git + +# Set the CKAN database connection string as an environment variable, plus set the process string for the HEALTHCHECK +ENV ckanext.xloader.jobs_db.uri="postgresql://ckandbuser:ckandbpassword@db/ckandb" +ENV WORKER_PROCESS="ckan -c /srv/app/ckan.ini jobs worker" + +# Customize these in the environment (.env) file if needed +ENV CKAN_SITE_URL=http://ckan-dev:5000 +ENV CKAN_CONTAINER_URL=http://ckan-dev:5000 +ENV CKAN__PLUGINS image_view text_view recline_view datastore xloader envvars + +# Set the working directory +WORKDIR ${APP_DIR} + +# Set up timezone +RUN echo ${TZ} > /etc/timezone + +# Set LC_ALL=en_US.UTF-8 will ensure that all locale-dependent operations in the current environment +# will use English language and United States cultural conventions with UTF-8 character encoding +ENV LC_ALL=en_US.UTF-8 + +# Set the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install system libraries +RUN apt-get install --no-install-recommends -y \ + apt-utils \ + git \ + libpq-dev \ + g++ \ + linux-headers-generic \ + libtool \ + wget \ + procps + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Copy setup file(s) to the image +COPY setup/start_background_worker.sh ${APP_DIR} + +# Install the CKAN application, the dependency packages for CKAN plus some confiquration +RUN cd ${SRC_DIR} && \ + pip3 install -e git+${GIT_URL}@${CKAN_TAG}#egg=ckan && \ + cd ckan && \ + cp who.ini ${APP_DIR} && \ + pip3 install --no-binary markdown -r requirements.txt && \ + # Install CKAN envvars to support loading config from environment variables + pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ + # Create and update CKAN config + ckan generate config ${CKAN_INI} && \ + ckan config-tool ${CKAN_INI} "beaker.session.secret = " && \ + ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" + +# Install the XLoader extension +RUN cd ${SRC_DIR} && \ + pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ + pip3 install -r ckanext-xloader/requirements.txt && \ + pip3 install -U requests[security] + +# Create a local user and group plus set up the storage path +RUN groupadd -g 92 ckan && \ + useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan && \ + mkdir -p ${CKAN_STORAGE_PATH} && \ + chown -R ckan:ckan ${CKAN_STORAGE_PATH} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD pgrep ${WORKER_PROCESS} > /dev/null || exit 1 + +#CMD ["/bin/bash"] +CMD ["/srv/app/start_background_worker.sh"] \ No newline at end of file diff --git a/xloader/Makefile b/xloader/Makefile new file mode 100644 index 0000000..7171918 --- /dev/null +++ b/xloader/Makefile @@ -0,0 +1,19 @@ +.PHONY: all help build build-all push +SHELL := /bin/bash +CKAN_VERSION=2.10.4 +XLOADER_VERSION=1.0.1 +CKAN_VERSION_MAJOR=$(shell echo $(CKAN_VERSION) | cut -d'.' -f1) +CKAN_VERSION_MINOR=$(shell echo $(CKAN_VERSION) | cut -d'.' -f2) +TAG_NAME="ckan/ckan-base-xloader:$(XLOADER_VERSION)" + +all: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +build: ## Build CKAN XLoader images , `make build` + echo "Building $(TAG_NAME) images" + docker build --build-arg="CKAN_TAG=ckan-$(CKAN_VERSION)" --build-arg="XLOADER_TAG=$(XLOADER_VERSION)" -t $(TAG_NAME) --no-cache . + +push: ## Push CKAN XLoader images to the DockerHub registry, `make push` + echo "Pushing $(TAG_NAME) image" + docker push $(TAG_NAME) \ No newline at end of file diff --git a/xloader/setup/start_background_worker.sh b/xloader/setup/start_background_worker.sh new file mode 100755 index 0000000..8d9878b --- /dev/null +++ b/xloader/setup/start_background_worker.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +if [[ $CKAN__PLUGINS == *"xloader"* ]]; then + # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) + echo "Setting a temporary value for ckan.xloader.api_token" + ckan config-tool $CKAN_INI ckan.xloader.api_token=xxx +fi + +if grep -qE "beaker.session.secret ?= ?$" ckan.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +echo "Set up ckan.xloader.api_token in the CKAN config file" + ckan config-tool $CKAN_INI "ckan.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + done +fi + +# Run the background worker +ckan -c $CKAN_INI jobs worker \ No newline at end of file