From 1dc7282a51ed37c2d379f7dec5e87e0e720a85c7 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 21 Aug 2024 11:49:13 +0200 Subject: [PATCH 1/4] Add ckan patch and minor version --- compose/2.10/docker-compose.yml | 4 +- compose/2.11/.ckan-env | 42 +++ compose/2.11/.env | 35 +++ compose/2.11/docker-compose.yml | 98 +++++++ compose/2.11/postgresql/Dockerfile | 13 + .../00_create_datastore.sh | 8 + .../20_postgis_permissions.sql | 2 + compose/2.11/solr8/ckan_init_solr.sh | 41 +++ images/ckan/2.10/Dockerfile | 4 +- images/ckan/2.10/Dockerfile.focal | 4 +- images/ckan/2.11/Dockerfile | 190 ++++++++++++++ images/ckan/2.11/Dockerfile.focal | 242 ++++++++++++++++++ images/ckan/2.11/patches/.gitkeep | 0 .../ckan/2.11/scripts/apply_ckan_patches.sh | 5 + images/ckan/2.11/setup/app/extra_scripts.sh | 4 + .../api/3/action/status_show/index.html | 25 ++ .../2.11/setup/app/maintenance/index.html | 25 ++ .../ckan/2.11/setup/app/maintenance/serve.py | 35 +++ images/ckan/2.11/setup/app/prerun.py | 228 +++++++++++++++++ images/ckan/2.11/setup/app/start_ckan.sh | 85 ++++++ images/ckan/2.11/setup/app/uwsgi.conf | 2 + images/ckan/2.11/setup/app/wsgi.py | 28 ++ 22 files changed, 1115 insertions(+), 5 deletions(-) create mode 100644 compose/2.11/.ckan-env create mode 100644 compose/2.11/.env create mode 100644 compose/2.11/docker-compose.yml create mode 100644 compose/2.11/postgresql/Dockerfile create mode 100644 compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh create mode 100644 compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql create mode 100755 compose/2.11/solr8/ckan_init_solr.sh create mode 100644 images/ckan/2.11/Dockerfile create mode 100644 images/ckan/2.11/Dockerfile.focal create mode 100644 images/ckan/2.11/patches/.gitkeep create mode 100755 images/ckan/2.11/scripts/apply_ckan_patches.sh create mode 100755 images/ckan/2.11/setup/app/extra_scripts.sh create mode 100644 images/ckan/2.11/setup/app/maintenance/api/3/action/status_show/index.html create mode 100644 images/ckan/2.11/setup/app/maintenance/index.html create mode 100644 images/ckan/2.11/setup/app/maintenance/serve.py create mode 100644 images/ckan/2.11/setup/app/prerun.py create mode 100755 images/ckan/2.11/setup/app/start_ckan.sh create mode 100644 images/ckan/2.11/setup/app/uwsgi.conf create mode 100644 images/ckan/2.11/setup/app/wsgi.py diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index 091a3e4..f8febcd 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -9,7 +9,9 @@ volumes: services: ckan: container_name: ckan - image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} + build: + context: ../../images/ckan/2.10 + dockerfile: Dockerfile networks: - frontend - backend diff --git a/compose/2.11/.ckan-env b/compose/2.11/.ckan-env new file mode 100644 index 0000000..b9f2598 --- /dev/null +++ b/compose/2.11/.ckan-env @@ -0,0 +1,42 @@ +# Runtime configuration of CKAN enabled through ckanext-envvars +# Information about how it works: https://github.com/okfn/ckanext-envvars +# Note that variables here take presedence over build/up time variables in .env + +# Set to true to disable CKAN from starting and serve a maintenance page +MAINTENANCE_MODE=false + +# General Settings +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 +CKAN_MAX_UPLOAD_SIZE_MB=20 +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +# CKAN Plugins +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +# CKAN requires storage path to be set in order for filestore to be enabled +CKAN__STORAGE_PATH=/srv/app/data +CKAN__WEBASSETS__PATH=/srv/app/data/webassets +# SYSADMIN settings, a sysadmin user is created automatically with the below credentials +CKAN_SYSADMIN_NAME=sysadmin +CKAN_SYSADMIN_PASSWORD=password +CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com + +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ +CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 +# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts + +# Solr configuration +CKAN_VERSION=2.10 +CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/.env b/compose/2.11/.env new file mode 100644 index 0000000..d8f5ef1 --- /dev/null +++ b/compose/2.11/.env @@ -0,0 +1,35 @@ +# Variables in this file will be used as build arguments when running +# docker-compose build and docker-compose up +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down -v +# docker-compose build +# docker-compose up -d + +# Database +POSTGRES_PASSWORD=ckan +POSTGRES_PORT=5432 +DATASTORE_READONLY_PASSWORD=datastore + +# CKAN +CKAN_VERSION=2.10.4 +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 +CKAN_MAX_UPLOAD_SIZE_MB=10 + +# Datapusher +DATAPUSHER_VERSION=0.0.17 +DATAPUSHER_MAX_CONTENT_LENGTH=10485760 +DATAPUSHER_CHUNK_SIZE=16384 +DATAPUSHER_CHUNK_INSERT_ROWS=250 +DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5005 + +# SOLR +CKAN_CORE_NAME=ckan + +# Redis +REDIS_VERSION=6.0.7 diff --git a/compose/2.11/docker-compose.yml b/compose/2.11/docker-compose.yml new file mode 100644 index 0000000..475b2e8 --- /dev/null +++ b/compose/2.11/docker-compose.yml @@ -0,0 +1,98 @@ +# docker-compose build && docker-compose up -d +version: "3.8" + +volumes: + ckan_data: + pg_data: + solr_data: + +services: + ckan: + container_name: ckan + build: + context: ../../images/ckan/2.11/ + dockerfile: Dockerfile + networks: + - frontend + - backend + depends_on: + - db + - solr + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + env_file: + - ./.ckan-env + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore + - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + + volumes: + - ckan_data:/srv/app/data + + datapusher: + container_name: datapusher + image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} + networks: + - frontend + - backend + ports: + - "8000:8000" + environment: + - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} + - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} + - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} + - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} + - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} + + + db: + container_name: db + build: + context: . + dockerfile: postgresql/Dockerfile + args: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + networks: + - backend + environment: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=/var/lib/postgresql/data/db + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "ckan"] + + solr: + container_name: solr + image: solr:8.11.1 + networks: + - backend + env_file: + - ./.ckan-env + environment: + - CKAN_CORE_NAME=${CKAN_CORE_NAME} + - CKAN_VERSION=${CKAN_VERSION} + volumes: + - solr_data:/var/solr + - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh + + redis: + container_name: redis + image: redis:${REDIS_VERSION} + networks: + - backend + +networks: + frontend: + backend: diff --git a/compose/2.11/postgresql/Dockerfile b/compose/2.11/postgresql/Dockerfile new file mode 100644 index 0000000..e5fcbb7 --- /dev/null +++ b/compose/2.11/postgresql/Dockerfile @@ -0,0 +1,13 @@ +FROM postgis/postgis:14-3.2-alpine + +# Allow connections; we don't map out any ports so only linked docker containers can connect +RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf + +# Customize default user/pass/db +ENV POSTGRES_DB ckan +ENV POSTGRES_USER ckan +ARG POSTGRES_PASSWORD +ARG DS_RO_PASS + +# Include datastore setup scripts +COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh new file mode 100644 index 0000000..ec1b3c3 --- /dev/null +++ b/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; + CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; + GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; +EOSQL \ No newline at end of file diff --git a/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql new file mode 100644 index 0000000..8b5348e --- /dev/null +++ b/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql @@ -0,0 +1,2 @@ +ALTER VIEW geometry_columns OWNER TO ckan; +ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/compose/2.11/solr8/ckan_init_solr.sh b/compose/2.11/solr8/ckan_init_solr.sh new file mode 100755 index 0000000..091450f --- /dev/null +++ b/compose/2.11/solr8/ckan_init_solr.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Initialize SOLR for CKAN by creating a ckan core +# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION +# Example: +# CKAN_CORE_NAME=ckan +# CKAN_VERSION=2.9.7 + +set -e + +MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1` +MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2` + +CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml + +echo "Check whether managed schema exists for CKAN $CKAN_VERSION" +if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then + echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..." + exit 1 +fi + +echo "Check whether SOLR is initialized for CKAN" +CORESDIR=/var/solr/data + +COREDIR="$CORESDIR/$CKAN_CORE_NAME" +if [ -d "$COREDIR" ]; then + echo "SOLR already initialized, skipping initialization" +else + echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION" + # init script for handling an empty /var/solr + /opt/docker-solr/scripts/init-var-solr + + # Precreate CKAN core + /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME + + # Replace the managed schema with CKANs schema + echo "Adding CKAN managed schema" + curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s + + echo "SOLR initialized" +fi diff --git a/images/ckan/2.10/Dockerfile b/images/ckan/2.10/Dockerfile index 3b78ed6..46924f1 100644 --- a/images/ckan/2.10/Dockerfile +++ b/images/ckan/2.10/Dockerfile @@ -4,11 +4,11 @@ FROM alpine:3.17.2 as ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.10.4 +ENV IMAGE_TAG=2.10.5 # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.10.4 +ENV GIT_BRANCH=ckan-2.10.5 # Set src dirs ENV SRC_DIR=/srv/app/src diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal index 84ec629..5428091 100644 --- a/images/ckan/2.10/Dockerfile.focal +++ b/images/ckan/2.10/Dockerfile.focal @@ -4,11 +4,11 @@ FROM ubuntu:focal-20210827 as ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.10.4-focal +ENV IMAGE_TAG=2.10.5-focal # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.10.4 +ENV GIT_BRANCH=ckan-2.10.5 # Set timezone ENV TZ=UTC diff --git a/images/ckan/2.11/Dockerfile b/images/ckan/2.11/Dockerfile new file mode 100644 index 0000000..7156368 --- /dev/null +++ b/images/ckan/2.11/Dockerfile @@ -0,0 +1,190 @@ +################## +### Build CKAN ### +################## +FROM alpine:3.17.2 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.11 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.11 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Packages to build CKAN requirements and plugins +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +# RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 + + +########################### +### Default-Extensions #### +########################### +FROM alpine:3.17.2 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM alpine:3.17.2 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apk add --no-cache \ + python3 \ + bash \ + git \ + gettext \ + curl \ + postgresql-client \ + libmagic \ + pcre \ + libxslt \ + libxml2 \ + tzdata \ + apache2-utils && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} + + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Set timezone + echo "UTC" > /etc/timezone && \ + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.11/Dockerfile.focal b/images/ckan/2.11/Dockerfile.focal new file mode 100644 index 0000000..ec2fc2d --- /dev/null +++ b/images/ckan/2.11/Dockerfile.focal @@ -0,0 +1,242 @@ +################## +### Build CKAN ### +################## +FROM ubuntu:focal-20210827 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10-focal + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.11 + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Set the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Instal apt-utils +RUN apt-get install --no-install-recommends -y \ + apt-utils + +# Packages to build CKAN requirements and plugins +RUN apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + libpq-dev \ + linux-headers-generic \ + gcc-10 \ + make \ + g++-10 \ + autoconf \ + automake \ + libtool \ + patch \ + libpcre3-dev \ + libpcre3 \ + python3-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Use gcc 10 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10 + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 + +########################### +### Default-Extensions #### +########################### +FROM ubuntu:focal-20210827 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.1 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + python3-dev + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM ubuntu:focal-20210827 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set the locale +RUN apt-get update && \ + apt-get install --no-install-recommends -y locales && \ + sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=${LC_ALL} && \ + rm -rf /var/lib/apt/lists/* + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gettext \ + curl \ + ca-certificates \ + libpq5 \ + git \ + postgresql-client \ + python3 \ + python3-distutils \ + libpython3.8 \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils && \ + rm -rf /var/lib/apt/lists/* && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 + +# Create a local user and group to run the app +RUN groupadd -g 92 ckan && \ + useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.11/patches/.gitkeep b/images/ckan/2.11/patches/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/images/ckan/2.11/scripts/apply_ckan_patches.sh b/images/ckan/2.11/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.11/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/ckan/2.11/setup/app/extra_scripts.sh b/images/ckan/2.11/setup/app/extra_scripts.sh new file mode 100755 index 0000000..80a70ef --- /dev/null +++ b/images/ckan/2.11/setup/app/extra_scripts.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# this is called before uwsgi is executed +# uset his to add extra scripts before ckan is started diff --git a/images/ckan/2.11/setup/app/maintenance/api/3/action/status_show/index.html b/images/ckan/2.11/setup/app/maintenance/api/3/action/status_show/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.11/setup/app/maintenance/api/3/action/status_show/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.11/setup/app/maintenance/index.html b/images/ckan/2.11/setup/app/maintenance/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.11/setup/app/maintenance/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.11/setup/app/maintenance/serve.py b/images/ckan/2.11/setup/app/maintenance/serve.py new file mode 100644 index 0000000..77f3bd4 --- /dev/null +++ b/images/ckan/2.11/setup/app/maintenance/serve.py @@ -0,0 +1,35 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +import os + +PORT = 5000 + +web_dir = os.path.join(os.path.dirname(__file__)) +os.chdir(web_dir) + + +def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler): + server_address = ("0.0.0.0", PORT) + httpd = server_class(server_address, handler_class) + print("Starting maintenance mode") + httpd.serve_forever() + + +if __name__ == "__main__": + run() + diff --git a/images/ckan/2.11/setup/app/prerun.py b/images/ckan/2.11/setup/app/prerun.py new file mode 100644 index 0000000..8fad1f7 --- /dev/null +++ b/images/ckan/2.11/setup/app/prerun.py @@ -0,0 +1,228 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from multiprocessing import connection + +import os +import sys +import subprocess +import psycopg2 +from sqlalchemy.engine.url import make_url +import urllib.request, urllib.error, urllib.parse, base64 +import re +import json +import time + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') + +RETRY = 5 + +def check_db_connection(retry=None): + + print('[prerun] Start check_db_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') + try: + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + + except psycopg2.Error as e: + print((str(e))) + print('[prerun] Unable to connect to the database...try again in a while.') + import time + time.sleep(10) + check_db_connection(retry = retry - 1) + else: + connection.close() + +def check_solr_connection(retry=None): + + print('[prerun] Start check_solr_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + username = os.environ.get('CKAN_SOLR_USER', '') + password = os.environ.get('CKAN_SOLR_PASSWORD', '') + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + if not username: + connection = urllib.request.urlopen(search_url) + else: + request = urllib.request.Request(search_url) + base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii')) + request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8')) + connection = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print('[prerun] Unable to connect to solr...try again in a while.') + import time + time.sleep(10) + check_solr_connection(retry = retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + sys.exit(1) + +def init_db(): + + print('[prerun] Start init_db...') + + db_command = ['ckan', '-c', ckan_ini, 'db', 'init'] + + print('[prerun] Initializing or upgrading db - start using ckan db init') + try: + # run init scripts + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + import time + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + print('[prerun] Initializing or upgrading db - finish') + + +def init_datastore(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] Skipping datastore initialization') + return + + datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore', + 'set-permissions'] + + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + cursor = connection.cursor() + + print('[prerun] Initializing datastore db - start') + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read() + perms_sql = perms_sql.decode('utf-8') + perms_sql = perms_sql.replace("@"+db_host, "") + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print('[prerun] Initializing datastore db - end') + print((datastore_perms.stdout.read())) + except psycopg2.Error as e: + print('[prerun] Could not initialize datastore') + print(e.decode('utf-8')) + + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + print('[prerun] Start create_sysadmin...') + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['ckan', '-c', ckan_ini, 'user', 'show', name] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')): + print('[prerun] Sysadmin user exists, skipping creation') + return + + # Create user + command = ['ckan', '-c', ckan_ini, 'user', 'add', + name, + 'password=' + password, + 'email=' + email] + + subprocess.call(command) + print(('[prerun] Created user {0}'.format(name))) + + # Make it sysadmin + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] + + subprocess.call(command) + print(('[prerun] Made user {0} a sysadmin'.format(name))) + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[prerun] Maintenance mode, skipping setup...') + else: + check_db_connection() + check_solr_connection() + init_db() + if os.environ.get('CKAN_DATASTORE_WRITE_URL'): + init_datastore() + create_sysadmin() diff --git a/images/ckan/2.11/setup/app/start_ckan.sh b/images/ckan/2.11/setup/app/start_ckan.sh new file mode 100755 index 0000000..0435142 --- /dev/null +++ b/images/ckan/2.11/setup/app/start_ckan.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) +ckan config-tool $APP_DIR/production.ini ckan.datapusher.api_token=xxx + +# Run any startup scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-entrypoint.d" ]] +then + for f in ${APP_DIR}/docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Add session secret from chart +if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then + echo "Not all environment variables are set. Generating sessions..." +else + echo "Setting session secrets from environment variables" + ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET" +fi + +if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini +then + echo "Setting secrets in ini file" + ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET" +fi + +echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers" +UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application" + +# Run the prerun script to init CKAN and create the default admin user +python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } + +echo "Set up ckan.datapusher.api_token in the CKAN config file" +ckan config-tool $APP_DIR/production.ini "ckan.datapusher.api_token=$(ckan -c $APP_DIR/production.ini user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" + +# Check if we are in maintenance mode and if yes serve the maintenance pages +if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi + +# Run any after prerun/init scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-afterinit.d" ]] +then + for f in ${APP_DIR}/docker-afterinit.d/*; do + case "$f" in + *.sh) echo "$0: Running after prerun init file $f"; . "$f" ;; + *.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start uwsgi with basicauth + uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start uwsgi + uwsgi $UWSGI_OPTS + fi +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/images/ckan/2.11/setup/app/uwsgi.conf b/images/ckan/2.11/setup/app/uwsgi.conf new file mode 100644 index 0000000..6321d6d --- /dev/null +++ b/images/ckan/2.11/setup/app/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/images/ckan/2.11/setup/app/wsgi.py b/images/ckan/2.11/setup/app/wsgi.py new file mode 100644 index 0000000..63f3a7c --- /dev/null +++ b/images/ckan/2.11/setup/app/wsgi.py @@ -0,0 +1,28 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# -*- coding: utf-8 -*- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), u'production.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) From c05df9465b5996bdf64fd837283e26a46df2e0d9 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 21 Aug 2024 12:08:43 +0200 Subject: [PATCH 2/4] Add CKAN github actions workflow --- .github/workflows/master_merge.yml | 65 ++++++++++++++++++++++++++++++ .github/workflows/pr_checks.yml | 50 +++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml index b521325..37f9bac 100644 --- a/.github/workflows/master_merge.yml +++ b/.github/workflows/master_merge.yml @@ -6,6 +6,71 @@ on: jobs: + build-ckan-2-11: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.CR_PAT }} + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: | + /tmp/.buildx-cache-alpine-2-11 + /tmp/.buildx-cache-ubuntu-2-11 + key: ${{ runner.os }}-buildx-2-11-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-2-11 + + - name: Get docker tag for Alpine image + id: alpine + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile)" >> $GITHUB_OUTPUT + + - name: Build and push CKAN 2.11 alpine + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile + push: true + tags: | + keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11 + + - name: Get docker tag for Ubuntu image + id: ubuntu + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.focal)" >> $GITHUB_OUTPUT + + - name: Build and push CKAN 2.11 ubuntu + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile.focal + push: true + tags: | + keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-11 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-11 + build-ckan-2-10: runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 0bad5a2..01507b7 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,6 +6,56 @@ on: jobs: + build-ckan-2-11: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: | + /tmp/.buildx-cache-alpine-2-11 + /tmp/.buildx-cache-ubuntu-2-11 + key: ${{ runner.os }}-buildx-2-11-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-2-11 + + - name: Get docker tag for Alpine image + id: alpine + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile)" >> $GITHUB_OUTPUT + + - name: Build CKAN 2.11 alpine + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile + push: false + tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11 + + - name: Get docker tag for Ubuntu image + id: ubuntu + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.focal)" >> $GITHUB_OUTPUT + + - name: Build CKAN 2.11 ubuntu + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile.focal + push: false + tags: keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-11 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-11 + build-args: | + --progress=plain + build-ckan-2-10: runs-on: ubuntu-20.04 steps: From 89a5e1a60275f824304631bd1286e16e3940285c Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 21 Aug 2024 14:24:15 +0200 Subject: [PATCH 3/4] Finish up changes for new patch version of CKAN 2.10.5 --- compose/2.10/.env | 2 +- compose/2.10/docker-compose.yml | 6 +----- images/ckan/2.10/Dockerfile | 4 ++-- images/ckan/2.10/Dockerfile.focal | 4 ++-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/compose/2.10/.env b/compose/2.10/.env index d8f5ef1..4556cfd 100644 --- a/compose/2.10/.env +++ b/compose/2.10/.env @@ -12,7 +12,7 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore # CKAN -CKAN_VERSION=2.10.4 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index f8febcd..c7f08a4 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -1,6 +1,4 @@ # docker-compose build && docker-compose up -d -version: "3.8" - volumes: ckan_data: pg_data: @@ -9,9 +7,7 @@ volumes: services: ckan: container_name: ckan - build: - context: ../../images/ckan/2.10 - dockerfile: Dockerfile + image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} networks: - frontend - backend diff --git a/images/ckan/2.10/Dockerfile b/images/ckan/2.10/Dockerfile index 46924f1..d4d27e5 100644 --- a/images/ckan/2.10/Dockerfile +++ b/images/ckan/2.10/Dockerfile @@ -1,7 +1,7 @@ ################## ### Build CKAN ### ################## -FROM alpine:3.17.2 as ckanbuild +FROM alpine:3.17.2 AS ckanbuild # Used by Github Actions to tag the image with ENV IMAGE_TAG=2.10.5 @@ -63,7 +63,7 @@ RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 ########################### ### Default-Extensions #### ########################### -FROM alpine:3.17.2 as extbuild +FROM alpine:3.17.2 AS extbuild # Set src dirs ENV SRC_DIR=/srv/app/src diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal index 5428091..8402695 100644 --- a/images/ckan/2.10/Dockerfile.focal +++ b/images/ckan/2.10/Dockerfile.focal @@ -1,7 +1,7 @@ ################## ### Build CKAN ### ################## -FROM ubuntu:focal-20210827 as ckanbuild +FROM ubuntu:focal-20210827 AS ckanbuild # Used by Github Actions to tag the image with ENV IMAGE_TAG=2.10.5-focal @@ -86,7 +86,7 @@ RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 ########################### ### Default-Extensions #### ########################### -FROM ubuntu:focal-20210827 as extbuild +FROM ubuntu:focal-20210827 AS extbuild # Set src dirs ENV SRC_DIR=/srv/app/src From d445972f755bb46a0daec595490e6392785bf1a7 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 21 Aug 2024 15:44:25 +0200 Subject: [PATCH 4/4] Add config for CKAN 2.11 --- compose/2.11/.ckan-env | 4 ++-- compose/2.11/.env | 3 ++- compose/2.11/docker-compose.yml | 2 -- images/ckan/2.11/Dockerfile | 14 +++++++------- images/ckan/2.11/Dockerfile.focal | 8 ++++---- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/compose/2.11/.ckan-env b/compose/2.11/.ckan-env index b9f2598..a329522 100644 --- a/compose/2.11/.ckan-env +++ b/compose/2.11/.ckan-env @@ -15,7 +15,7 @@ CKAN___BEAKER__SESSION__SECRET=CHANGE_ME CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME # CKAN Plugins -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__PLUGINS=envvars activity image_view # CKAN requires storage path to be set in order for filestore to be enabled CKAN__STORAGE_PATH=/srv/app/data CKAN__WEBASSETS__PATH=/srv/app/data/webassets @@ -38,5 +38,5 @@ CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0l # CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts # Solr configuration -CKAN_VERSION=2.10 +CKAN_VERSION=2.11 CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/.env b/compose/2.11/.env index d8f5ef1..394a5fd 100644 --- a/compose/2.11/.env +++ b/compose/2.11/.env @@ -12,7 +12,7 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore # CKAN -CKAN_VERSION=2.10.4 +CKAN_VERSION=2.11.0 CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 @@ -27,6 +27,7 @@ DATAPUSHER_DOWNLOAD_TIMEOUT=30 DATAPUSHER_SSL_VERIFY=False DATAPUSHER_REWRITE_RESOURCES=True DATAPUSHER_REWRITE_URL=http://ckan:5005 +DATA_PUSHER_SSL_VERIFY=False # SOLR CKAN_CORE_NAME=ckan diff --git a/compose/2.11/docker-compose.yml b/compose/2.11/docker-compose.yml index 475b2e8..18e9927 100644 --- a/compose/2.11/docker-compose.yml +++ b/compose/2.11/docker-compose.yml @@ -1,6 +1,4 @@ # docker-compose build && docker-compose up -d -version: "3.8" - volumes: ckan_data: pg_data: diff --git a/images/ckan/2.11/Dockerfile b/images/ckan/2.11/Dockerfile index 7156368..81c6c1d 100644 --- a/images/ckan/2.11/Dockerfile +++ b/images/ckan/2.11/Dockerfile @@ -1,14 +1,14 @@ ################## ### Build CKAN ### ################## -FROM alpine:3.17.2 as ckanbuild +FROM alpine:3.17.2 AS ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.11 +ENV IMAGE_TAG=2.11.0 # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.11 +ENV GIT_BRANCH=ckan-2.11.0 # Set src dirs ENV SRC_DIR=/srv/app/src @@ -63,7 +63,7 @@ RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 ########################### ### Default-Extensions #### ########################### -FROM alpine:3.17.2 as extbuild +FROM alpine:3.17.2 AS extbuild # Set src dirs ENV SRC_DIR=/srv/app/src @@ -74,7 +74,7 @@ ENV DEFAULT_EXTENSIONS envvars # Locations and tags, please use specific tags or revisions ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars -ENV ENVVARS_GIT_BRANCH=0.0.2 +ENV ENVVARS_GIT_BRANCH=v0.0.4 RUN apk add --no-cache \ python3 \ @@ -106,7 +106,7 @@ ENV CKAN_DIR=${SRC_DIR}/ckan ENV DATA_DIR=/srv/app/data ENV PIP_SRC=${SRC_DIR} ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher +ENV CKAN__PLUGINS envvars activity image_view # Install necessary packages to run CKAN RUN apk add --no-cache \ @@ -146,7 +146,7 @@ WORKDIR ${CKAN_DIR} # Install CKAN RUN pip install -e /srv/app/src/ckan && \ - cp who.ini ${APP_DIR} && \ + # cp who.ini ${APP_DIR} removed this because there was not in 2.11.0 pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ # Install default CKAN extensions pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ diff --git a/images/ckan/2.11/Dockerfile.focal b/images/ckan/2.11/Dockerfile.focal index ec2fc2d..83ffd5b 100644 --- a/images/ckan/2.11/Dockerfile.focal +++ b/images/ckan/2.11/Dockerfile.focal @@ -4,11 +4,11 @@ FROM ubuntu:focal-20210827 as ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.10-focal +ENV IMAGE_TAG=2.11.0-focal # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.11 +ENV GIT_BRANCH=ckan-2.11.0 # Set timezone ENV TZ=UTC @@ -97,7 +97,7 @@ ENV DEFAULT_EXTENSIONS envvars # Locations and tags, please use specific tags or revisions ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars -ENV ENVVARS_GIT_BRANCH=0.0.1 +ENV ENVVARS_GIT_BRANCH=v0.0.4 RUN apt-get update && \ apt-get install --no-install-recommends -y \ @@ -152,7 +152,7 @@ ENV CKAN_DIR=${SRC_DIR}/ckan ENV DATA_DIR=/srv/app/data ENV PIP_SRC=${SRC_DIR} ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher +ENV CKAN__PLUGINS envvars activity image_view # Install necessary packages to run CKAN RUN apt-get update && \