Merge pull request #134 from keitaroinc/new-release

New release 2.10.5 and 2.11.0

.github/workflows/master_merge.yml

@@ -6,6 +6,71 @@ on:
 
 
 jobs:
+  build-ckan-2-11:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.CR_PAT }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: |
+            /tmp/.buildx-cache-alpine-2-11
+            /tmp/.buildx-cache-ubuntu-2-11
+          key: ${{ runner.os }}-buildx-2-11-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-2-11
+
+      - name: Get docker tag for Alpine image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile)" >> $GITHUB_OUTPUT
+
+      - name: Build and push CKAN 2.11 alpine
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.11
+          file: ./images/ckan/2.11/Dockerfile
+          push: true
+          tags: |
+            keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+            ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11
+
+      - name: Get docker tag for Ubuntu image
+        id: ubuntu
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.focal)" >> $GITHUB_OUTPUT
+
+      - name: Build and push CKAN 2.11 ubuntu
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.11
+          file: ./images/ckan/2.11/Dockerfile.focal
+          push: true
+          tags: |
+            keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }}
+            ghcr.io/keitaroinc/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-11
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-11
+
   build-ckan-2-10:
     runs-on: ubuntu-20.04
     steps:

.github/workflows/pr_checks.yml

@@ -6,6 +6,56 @@ on:
 
 
 jobs:
+  build-ckan-2-11:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: |
+            /tmp/.buildx-cache-alpine-2-11
+            /tmp/.buildx-cache-ubuntu-2-11
+          key: ${{ runner.os }}-buildx-2-11-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-2-11
+
+      - name: Get docker tag for Alpine image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile)" >> $GITHUB_OUTPUT
+
+      - name: Build CKAN 2.11 alpine
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.11
+          file: ./images/ckan/2.11/Dockerfile
+          push: false
+          tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11
+
+      - name: Get docker tag for Ubuntu image
+        id: ubuntu
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.focal)" >> $GITHUB_OUTPUT
+
+      - name: Build CKAN 2.11 ubuntu
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.11
+          file: ./images/ckan/2.11/Dockerfile.focal
+          push: false
+          tags: keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-11
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-11
+          build-args: |
+           --progress=plain
+
   build-ckan-2-10:
     runs-on: ubuntu-20.04
     steps:

compose/2.10/.env

@@ -12,7 +12,7 @@ POSTGRES_PORT=5432
 DATASTORE_READONLY_PASSWORD=datastore
 
 # CKAN
-CKAN_VERSION=2.10.4
+CKAN_VERSION=2.10.5
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://localhost:5000
 CKAN_PORT=5000

compose/2.10/docker-compose.yml

@@ -1,6 +1,4 @@
 # docker-compose build && docker-compose up -d
-version: "3.8"
-
 volumes:
   ckan_data:
   pg_data:

compose/2.11/.ckan-env

@@ -0,0 +1,42 @@
+# Runtime configuration of CKAN enabled through ckanext-envvars
+# Information about how it works: https://github.com/okfn/ckanext-envvars
+# Note that variables here take presedence over build/up time variables in .env
+
+# Set to true to disable CKAN from starting and serve a maintenance page
+MAINTENANCE_MODE=false
+
+# General Settings
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=20
+CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
+# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
+CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
+# CKAN Plugins
+CKAN__PLUGINS=envvars activity image_view
+# CKAN requires storage path to be set in order for filestore to be enabled
+CKAN__STORAGE_PATH=/srv/app/data
+CKAN__WEBASSETS__PATH=/srv/app/data/webassets
+# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
+CKAN_SYSADMIN_NAME=sysadmin
+CKAN_SYSADMIN_PASSWORD=password
+[email protected]
+
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0
+# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts
+
+# Solr configuration
+CKAN_VERSION=2.11
+CKAN_CORE_NAME=ckan

compose/2.11/.env

@@ -0,0 +1,36 @@
+# Variables in this file will be used as build arguments when running
+# docker-compose build and docker-compose up
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down -v
+# docker-compose build
+# docker-compose up -d
+
+# Database
+POSTGRES_PASSWORD=ckan
+POSTGRES_PORT=5432
+DATASTORE_READONLY_PASSWORD=datastore
+
+# CKAN
+CKAN_VERSION=2.11.0
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=10
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+DATAPUSHER_MAX_CONTENT_LENGTH=10485760
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5005
+DATA_PUSHER_SSL_VERIFY=False
+
+# SOLR
+CKAN_CORE_NAME=ckan
+
+# Redis
+REDIS_VERSION=6.0.7

compose/2.11/docker-compose.yml

@@ -0,0 +1,96 @@
+# docker-compose build && docker-compose up -d
+volumes:
+  ckan_data:
+  pg_data:
+  solr_data:
+
+services:
+  ckan:
+    container_name: ckan
+    build:
+      context: ../../images/ckan/2.11/
+      dockerfile: Dockerfile
+    networks:
+      - frontend
+      - backend
+    depends_on:
+      - db
+      - solr
+    ports:
+      - "0.0.0.0:${CKAN_PORT}:5000"
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
+      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
+      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
+      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+      - CKAN_REDIS_URL=redis://redis:6379/1
+      - CKAN_SITE_URL=${CKAN_SITE_URL}
+      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+
+    volumes:
+      - ckan_data:/srv/app/data
+
+  datapusher:
+    container_name: datapusher
+    image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
+    networks:
+      - frontend
+      - backend
+    ports:
+      - "8000:8000"
+    environment:
+      - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
+      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
+      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
+      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
+
+
+  db:
+    container_name: db
+    build:
+      context: .
+      dockerfile: postgresql/Dockerfile
+      args:
+        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    networks:
+      - backend
+    environment:
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - PGDATA=/var/lib/postgresql/data/db
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "ckan"]
+
+  solr:
+    container_name: solr
+    image: solr:8.11.1
+    networks:
+      - backend
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
+      - CKAN_VERSION=${CKAN_VERSION}
+    volumes:
+      - solr_data:/var/solr
+      - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
+
+  redis:
+    container_name: redis
+    image: redis:${REDIS_VERSION}
+    networks:
+      - backend
+
+networks:
+  frontend:
+  backend:

compose/2.11/postgresql/Dockerfile

@@ -0,0 +1,13 @@
+FROM postgis/postgis:14-3.2-alpine
+
+# Allow connections; we don't map out any ports so only linked docker containers can connect
+RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
+
+# Customize default user/pass/db
+ENV POSTGRES_DB ckan
+ENV POSTGRES_USER ckan
+ARG POSTGRES_PASSWORD
+ARG DS_RO_PASS
+
+# Include datastore setup scripts
+COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d

compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+    CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
+    CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
+    GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
+EOSQL

compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql

@@ -0,0 +1,2 @@
+ALTER VIEW geometry_columns OWNER TO ckan;
+ALTER TABLE spatial_ref_sys OWNER TO ckan;

compose/2.11/solr8/ckan_init_solr.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Initialize SOLR for CKAN by creating a ckan core
+# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
+# Example:
+#   CKAN_CORE_NAME=ckan
+#   CKAN_VERSION=2.9.7
+
+set -e
+
+MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1`
+MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2`
+
+CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml
+
+echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
+if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
+  echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
+  exit 1
+fi
+
+echo "Check whether SOLR is initialized for CKAN"
+CORESDIR=/var/solr/data
+
+COREDIR="$CORESDIR/$CKAN_CORE_NAME"
+if [ -d "$COREDIR" ]; then
+    echo "SOLR already initialized, skipping initialization"
+else
+    echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
+    # init script for handling an empty /var/solr
+    /opt/docker-solr/scripts/init-var-solr
+
+    # Precreate CKAN core
+    /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME
+
+    # Replace the managed schema with CKANs schema
+    echo "Adding CKAN managed schema"
+    curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
+
+    echo "SOLR initialized"
+fi