Merge branch 'getredash:master' into master

.ci/docker-compose.cypress.yml

@@ -3,8 +3,7 @@ x-redash-service: &redash-service
   build:
     context: ../
     args:
-      skip_dev_deps: "true"
-      skip_ds_deps: "true"
+      install_groups: "main"
       code_coverage: ${CODE_COVERAGE}
 x-redash-environment: &redash-environment
   REDASH_LOG_LEVEL: "INFO"

.ci/docker_build

@@ -1,21 +1,39 @@
 #!/bin/bash
+
+# This script only needs to run on the main Redash repo
+
+if [ "${GITHUB_REPOSITORY}" != "getredash/redash" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the main Redash repository"
+	exit 0
+fi
+
+if [ "${GITHUB_REF_NAME}" != "master" ] && [ "${GITHUB_REF_NAME}" != "preview-image" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the 'master' nor 'preview-image' branch"
+	exit 0
+fi
+
+if [ "x${DOCKER_USER}" = "x" ] || [ "x${DOCKER_PASS}" = "x" ]; then
+	echo "Skipping image build for Docker Hub, as the login details aren't available"
+	exit 0
+fi
+
 set -e
 VERSION=$(jq -r .version package.json)
-VERSION_TAG=$VERSION.b$CIRCLE_BUILD_NUM
+VERSION_TAG="$VERSION.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}"
 
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
-docker login -u $DOCKER_USER -p $DOCKER_PASS
+docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
 
-if [ $CIRCLE_BRANCH = master ] || [ $CIRCLE_BRANCH = preview-image ]
-then
-    docker build --build-arg skip_dev_deps=true -t redash/redash:preview -t redash/preview:$VERSION_TAG .
-    docker push redash/redash:preview
-    docker push redash/preview:$VERSION_TAG
-else
-    docker build --build-arg skip_dev_deps=true -t redash/redash:$VERSION_TAG .
-    docker push redash/redash:$VERSION_TAG
-fi
+DOCKERHUB_REPO="redash/redash"
+DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}"
+
+# Build the docker container
+docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} .
+
+# Push the container to the preview build locations
+docker push "${DOCKERHUB_REPO}:preview"
+docker push "redash/preview:${VERSION_TAG}"
 
-echo "Built: $VERSION_TAG"
+echo "Built: ${VERSION_TAG}"

.ci/update_version

@@ -1,6 +1,6 @@
 #!/bin/bash
 VERSION=$(jq -r .version package.json)
-FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}
 
-sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '$FULL_VERSION'/" redash/__init__.py
-sed -i "s/dev/$CIRCLE_SHA1/" client/app/version.json
+sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py
+sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json

.github/workflows/ci.yml

@@ -16,10 +16,9 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
-      - run: sudo pip install flake8==6.1.0 black==23.1.0 isort==5.12.0
-      - run: flake8 .
+      - run: sudo pip install black==23.1.0 ruff==0.0.287
+      - run: ruff check .
       - run: black --check .
-      - run: isort --check-only --diff .
 
   backend-unit-tests:
     runs-on: ubuntu-22.04
@@ -36,7 +35,7 @@ jobs:
       - name: Build Docker Images
         run: |
           set -x
-          docker-compose build --build-arg test_all_deps=true --build-arg skip_frontend_build=true
+          docker-compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
           docker-compose up -d
           sleep 10
       - name: Create Test Database
@@ -128,7 +127,7 @@ jobs:
       - name: Enable Code Coverage Report For Master Branch
         if: endsWith(github.ref, '/master')
         run: |
-          echo "CODE_COVERAGE=true" >> $GITHUB_ENV
+          echo "CODE_COVERAGE=true" >> "$GITHUB_ENV"
       - name: Install Dependencies
         run: |
           npm install --global --force [email protected]
@@ -151,3 +150,79 @@ jobs:
         with:
           name: coverage
           path: coverage
+
+  build-skip-check:
+    runs-on: ubuntu-22.04
+    outputs:
+      skip: ${{ steps.skip-check.outputs.skip }}
+    steps:
+      - name: Skip?
+        id: skip-check
+        run: |
+          if [[ "${{ vars.DOCKER_USER }}" == '' ]] || [[ "${{ secrets.DOCKER_PASS }}" == '' ]]; then
+            echo skip=true >> "$GITHUB_OUTPUT"
+          else
+            echo skip=false >> "$GITHUB_OUTPUT"
+          fi
+
+  build-docker-image:
+    runs-on: ubuntu-22.04
+    needs:
+      - backend-unit-tests
+      - frontend-unit-tests
+      - frontend-e2e-tests
+      - build-skip-check
+    if: ${{ needs.build-skip-check.outputs.skip == false}}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+      - name: Install Dependencies
+        run: |
+          npm install --global --force [email protected]
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+
+      - name: Set up QEMU
+        timeout-minutes: 1
+        uses: docker/[email protected]
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ vars.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+
+      - name: Bump version
+        id: version
+        run: |
+          set -x
+          .ci/update_version
+          VERSION=$(jq -r .version package.json)
+          VERSION_TAG="${VERSION}.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}"
+          echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push preview image to Docker Hub
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: |
+            redash/redash:preview
+            redash/preview:${{ steps.version.outputs.VERSION_TAG }}
+          context: .
+          build-args: |
+            test_all_deps=true
+          cache-from: type=ghq
+          cache-to: type=gha,mode=max
+        env:
+          DOCKER_CONTENT_TRUST: true
+
+      - name: "Failure: output container logs to console"
+        if: failure()
+        run: docker-compose logs

.github/workflows/periodic-snapshot.yml

@@ -0,0 +1,27 @@
+name: Periodic Snapshot
+
+# 10 minutes after midnight on the first of every month
+on:
+  schedule:
+  - cron: "10 0 1 * *"
+
+permissions:
+  contents: write
+
+jobs:
+  bump-version-and-tag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          date="$(date +%y.%m).0-dev"
+          gawk -i inplace -F: -v q=\" -v tag=$date '/^  "version": / { print $1 FS, q tag q ","; next} { print }' package.json
+          gawk -i inplace -F= -v q=\" -v tag=$date '/^__version__ =/ { print $1 FS, q tag q; next} { print }' redash/__init__.py
+          gawk -i inplace -F= -v q=\" -v tag=$date '/^version =/ { print $1 FS, q tag q; next} { print }' pyproject.toml
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add package.json redash/__init__.py pyproject.toml
+          git commit -m "Snapshot: ${date}"
+          git push origin
+          git tag $date
+          git push origin $date

.pre-commit-config.yaml

@@ -1,19 +1,10 @@
 repos:
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
   - repo: https://github.com/psf/black
     rev: 23.1.0
     hooks:
       - id: black
         language_version: python3
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        exclude: "migration/.*|.git|viz-lib|node_modules|migrations|bin/upgrade"
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.287"
     hooks:
-      - id: requirements-txt-fixer
+      - id: ruff

Dockerfile

@@ -29,18 +29,12 @@ FROM python:3.8-slim-buster
 
 EXPOSE 5000
 
-# Controls whether to install extra dependencies needed for all data sources.
-ARG skip_ds_deps
-# Controls whether to install dev dependencies.
-ARG skip_dev_deps
-# Controls whether to install all dependencies for testing.
-ARG test_all_deps
-
 RUN useradd --create-home redash
 
 # Ubuntu packages
 RUN apt-get update && \
   apt-get install -y --no-install-recommends \
+  pkg-config \
   curl \
   gnupg \
   build-essential \
@@ -84,24 +78,18 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
 
 WORKDIR /app
 
-# Disable PIP Cache and Version Check
-ENV PIP_DISABLE_PIP_VERSION_CHECK=1
-ENV PIP_NO_CACHE_DIR=1
-
-RUN pip install pip==23.1.2;
-
-# We first copy only the requirements file, to avoid rebuilding on every file change.
-COPY requirements_all_ds.txt ./
-RUN if [ "x$skip_ds_deps" = "x" ] ; then cat requirements_all_ds.txt | sed -e '/^\s*#.*$/d' -e '/^\s*$/d' | xargs -n 1 pip install || true ; else echo "Skipping pip install -r requirements_all_ds.txt" ; fi
-
-
-COPY requirements_dev.txt ./
-RUN if [ "x$skip_dev_deps" = "x" ] ; then pip install -r requirements_dev.txt ; fi
+ENV POETRY_VERSION=1.6.1
+ENV POETRY_HOME=/etc/poetry
+ENV POETRY_VIRTUALENVS_CREATE=false
+RUN curl -sSL https://install.python-poetry.org | python3 -
 
-COPY requirements.txt ./
-RUN pip install -r requirements.txt
+COPY pyproject.toml poetry.lock ./
 
-RUN if [ "x$test_all_deps" != "x" ] ; then pip3 install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt ; fi
+ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi"
+# for LDAP authentication, install with `ldap3` group
+# disabled by default due to GPL license conflict
+ARG install_groups="main,all_ds,dev"
+RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS
 
 COPY --chown=redash . /app
 COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist

Makefile

@@ -34,7 +34,8 @@ tests:
 	docker-compose run server tests
 
 lint:
-	./bin/flake8_tests.sh
+	ruff check .
+	black --check . --diff
 
 backend-unit-tests: up test_db
 	docker-compose run --rm --name tests server tests
@@ -43,15 +44,15 @@ frontend-unit-tests:
 	CYPRESS_INSTALL_BINARY=0 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 yarn --frozen-lockfile
 	yarn test
 
-test: lint backend-unit-tests frontend-unit-tests
+test: backend-unit-tests frontend-unit-tests lint
 
-build: 
+build:
 	yarn build
 
-watch: 
+watch:
 	yarn watch
 
-start: 
+start:
 	yarn start
 
 redis-cli:

README.md

@@ -90,6 +90,7 @@ Redash supports more than 35 SQL and NoSQL [data sources](https://redash.io/help
 - SPARQL
 - SQLite
 - TiDB
+- Tinybird
 - TreasureData
 - Trino
 - Uptycs