Make different artifacts.go included in the release for different env…

…ironments.

Signed-off-by: zeroalphat <[email protected]>

.circleci/config.yml

@@ -133,15 +133,12 @@ jobs:
       - setup-tools
       - run: time make deb SUDO="" FAKEROOT=
       - run: time make tools SUDO="" FAKEROOT=
-      - run:
-          command: |
-            if [ "${CIRCLE_BRANCH}" != "main" ]; then
-              go install ./pkg/generate-artifacts/
-              generate-artifacts > /tmp/artifacts.go
-              diff -u artifacts.go /tmp/artifacts.go
-            fi
 
   build_release:
+    parameters:
+      tags:
+        type: string
+        default: ""
     docker:
       - image: ghcr.io/cybozu/golang:1.22-jammy
       - image: ghcr.io/cybozu/etcd:3.5
@@ -156,12 +153,12 @@ jobs:
           name: Build deb package
           command: |
             VERSION=$(echo $CIRCLE_TAG | sed 's/^[^-]*-//')
-            time make deb SUDO="" DEBBUILD_FLAGS="-Zxz" VERSION="$VERSION" TAGS=release FAKEROOT=
+            time make deb SUDO="" DEBBUILD_FLAGS="-Zxz" VERSION="$VERSION" TAGS=<< parameters.tags >> FAKEROOT=
       - run:
           name: Build neco-operation-cli packages
           command: |
             VERSION=$(echo $CIRCLE_TAG | sed 's/^[^-]*-//')
-            time make tools SUDO="" DEBBUILD_FLAGS="-Zxz" VERSION="$VERSION" TAGS=release FAKEROOT=
+            time make tools SUDO="" DEBBUILD_FLAGS="-Zxz" VERSION="$VERSION" TAGS=<< parameters.tags >> FAKEROOT=
       - persist_to_workspace:
           root: .
           paths:
@@ -262,35 +259,27 @@ jobs:
             . ./bin/env
             $GCLOUD compute instances delete ${INSTANCE_NAME} --zone ${ZONE}
 
-  generate-artifacts:
+  generate-and-compare-artifacts:
     docker:
       - image: ghcr.io/cybozu/golang:1.22-jammy
     steps:
       - checkout
-      - run: go install ./pkg/generate-artifacts/
-      - run: generate-artifacts --release > artifacts_release.go
-      - persist_to_workspace:
-          root: .
-          paths:
-            - "artifacts_release.go"
+      - run: make update-artifacts
+      - run: 
+          name: Compare artifacts
+          command: |
+            diffs=$(git diff --name-only main)
+            if [ "$diffs" = "" ]; then printf "%s\n" "$diffs"; exit 0; fi
 
   dctest-functions-release:
     docker:
       - image: google/cloud-sdk
     working_directory: /work
     steps:
       - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Configure GIT
-          command: |
-            git config --global user.email "[email protected]"
-            git config --global user.name "cybozu-neco"
-      # Check difference between [`release` branch] and [`main` branch + `artifacts_release.go`]
+      # Check difference between [`release` branch] and [`main` branch ]
       - run: git checkout release
       - run: git merge --no-commit origin/main
-      - run: cp /tmp/workspace/artifacts_release.go .
       - run:
           name: Check diff
           command: |
@@ -328,17 +317,9 @@ jobs:
     working_directory: /work
     steps:
       - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Configure GIT
-          command: |
-            git config --global user.email "[email protected]"
-            git config --global user.name "cybozu-neco"
       # For testing on equal terms with "dctest-functions-release", check out in the same way as it.
       - run: git checkout release
       - run: git merge --no-commit origin/main
-      - run: cp /tmp/workspace/artifacts_release.go .
       - restore-tools
       - run-dctest-bootstrap:
           tag: release
@@ -369,17 +350,14 @@ jobs:
           fingerprints:
             - "1d:a7:0e:6e:a6:aa:9c:5e:d7:be:e1:59:3f:ed:68:f3"
       - checkout
-      - attach_workspace:
-          at: /tmp/workspace
       - run:
           name: Configure GIT
           command: |
             git config --global user.email "[email protected]"
             git config --global user.name "cybozu-neco"
-      # Check difference between [`release` branch] and [`main` branch + `artifacts_release.go`]
+      # Check difference between [`release` branch] and [`main` branch ]
       - run: git checkout release
       - run: git merge --no-commit origin/main
-      - run: cp /tmp/workspace/artifacts_release.go .
       - run:
           name: Check diff
           command: |
@@ -429,40 +407,60 @@ workflows:
   release:
     jobs:
       - build_release:
+          tags: ""
           filters:
             branches:
               ignore: /.*/
             tags:
-              only: /^(release|test)-.*/
+              only: /^test-.*/
       - deploy_github:
           requires:
             - build_release
           filters:
             branches:
               ignore: /.*/
             tags:
-              only: /^(release|test)-.*/
+              only: /^test-.*/
+      - build_release:
+          tags: "release"
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^(release)-.*/
+      - deploy_github:
+          requires:
+            - build_release
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^(release)-.*/
 
   # Testing `main` branch and update "release" branch.
   update-release:
     jobs:
-      - generate-artifacts:
-          filters:
-            branches:
-              only:
-                - main
-      - dctest-functions-release:
-          requires:
-            - generate-artifacts
-      - dctest-upgrade-release:
-          requires:
-            - generate-artifacts
+      - dctest-functions-release
+      - dctest-upgrade-release
       - update-release:
           requires:
             - dctest-functions-release
             - dctest-upgrade-release
 
-  # For updating `artifact_release.go` regularly, run the same job as `main` workflow.
+  # triggered weekdays at 15:00 UTC(00:00 UTC+9) on the main branch, 
+  # generates new artifacts and compares them with existing ones to highlight any discrepancies. 
+  daily-check-generate-and-compare-artifacts:
+    triggers:
+      - schedule:
+          cron: "0 15 * * 1-5"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - generate-and-compare-artifacts
+
+  # Run daily tests for the `main` branch.
   daily:
     triggers:
       - schedule:
@@ -472,17 +470,6 @@ workflows:
               only:
                 - main
     jobs:
-      - generate-artifacts
-      - dctest-functions-release:
-          requires:
-            - generate-artifacts
-      - dctest-upgrade-release:
-          requires:
-            - generate-artifacts
-      - dctest-reboot-worker:
-          requires:
-            - generate-artifacts
-      - update-release:
-          requires:
-            - dctest-functions-release
-            - dctest-upgrade-release
+      - dctest-functions-release
+      - dctest-upgrade-release
+      - dctest-reboot-worker

Makefile

@@ -217,6 +217,17 @@ gcp-deb: setup-files-for-deb
 .PHONY: git-neco
 git-neco:
 	go install ./pkg/git-neco
+
+.PHONY: setup-generate-artifacts
+setup-generate-artifacts:
+	GOBIN=$(BIN_DIR) go install ./pkg/generate-artifacts
+
+.PHONY: update-artifacts
+update-artifacts: setup-generate-artifacts
+	$(BIN_DIR)/generate-artifacts --ignore-file artifacts_ignore.yaml > /tmp/artifacts.go
+	cp /tmp/artifacts.go artifacts.go
+	$(BIN_DIR)/generate-artifacts --release --ignore-file artifacts_prod_ignore.yaml > /tmp/artifacts_prod.go
+	cp /tmp/artifacts_prod.go artifacts_prod.go
 
 .PHONY: setup
 setup:

bin/.gitignore

@@ -1,2 +1,3 @@
 /kustomize
 /helm
+/generate-artifacts

docs/artifacts.md

@@ -4,24 +4,22 @@ Artifacts
 [Artifacts](../artifacts.go) is the collection of components tested in [dctest](../dctest/), which consists of container images, debian package, and Flatcar image.
 
 There are two artifacts files.  When generating neco binaries and the neco deb package, `artifacts.go` is used
-by default.  `artifacts_release.go` is used only when `release` build tag is given.
+by default.  `artifacts_prod.go` is used only when `release` build tag is given.
 
 - [artifacts.go](../artifacts.go)
 
     This file describes artifacts to be tested in development.
     It should be updated with `generate-artifacts` command.
     You may edit the file **ONLY IF** you would like to test different combination of components.
 
-- [artifacts_release.go](../artifacts.go)
+- [artifacts_prod.go](../artifacts.go)
 
-    This file is for CI.  The CI job updates this file by using
-    `generate-artifacts --release`.
+    This file describes artifacts to be tested in production.
+    It should be updated with `generate-artifacts --release` command.
 
     The developers are PROHIBITED to modify this file.  This is to prevent
     merge conflicts in CI flow.
 
-    This file exists only in the `release` branch.
-
 ## How to handle prerelease versions
 
 As described above, you should not edit artifacts files manually in general.
@@ -30,9 +28,6 @@ to prepare for the update of the component.
 A component with prerelease version information, i.e. x.y.z-\<prerelease\>,
 does not get included in artifacts files by `generate-artifacts`.
 So if you want to use such a component, include it manually.
-Though you can edit `artifacts.go` in this case, you must not merge it into
-the main branch anyway, because that component does not get included in
-`artifacts_release.go` and CI on `release` branch will fail.
 Instead, after you confirm that neco can accept a component of x.y.z-\<prerelease\>,
 release the component as x.y.z and include it in artifacts files by `generate-artifacts`.
 

docs/cicd.md

@@ -16,11 +16,10 @@ CI flow
 
 ### After `main` merge
 
-1. Run `generate-artifacts` to retrieve latest version of components, then generate `artifacts_release.go`.
-2. Check out `main` branch then merge changes into `release` branch.
-3. Run [dctest][] but fewer test cases from regular dctest.
-4. If dctest is passed, [cybozu-neco][] pushes changes to `release` branch to remote branch in [github.com/cybozu-go/neco](https://github.com/cybozu-go/neco).
-5. Also, [cybozu-neco][] also applies and pushes a tag `test-YYYY.MM.DD-UNIQUE_ID` to the remote.
+1. Check out `main` branch then merge changes into `release` branch.
+2. Run [dctest][] but fewer test cases from regular dctest.
+3. If dctest is passed, [cybozu-neco][] pushes changes to `release` branch to remote branch in [github.com/cybozu-go/neco](https://github.com/cybozu-go/neco).
+4. Also, [cybozu-neco][] also applies and pushes a tag `test-YYYY.MM.DD-UNIQUE_ID` to the remote.
 
 Regular test cases of dctest are also run in parallel.
 

docs/dctest.md

@@ -116,7 +116,7 @@ This variable makes sense only when `SUITE=bootstrap` is specified.
 You can choose the list of artifacts by specifying `TAGS` make variable,
 though non-default value is only for CI.
 The default is to use `artifacts.go`.
-Specify `TAGS=release` in the release branch to use `artifacts_release.go`.
+Specify `TAGS=release` to use `artifacts_prod.go`.
 
 `make test` accepts this variable.
 

docs/generate-artifacts.md

@@ -3,7 +3,7 @@ generate-artifacts
 
 `generate-artifacts` is a command-line tool to generate `artifacts.go`
 which is a collection of latest components. This is used while
-development or CI before build neco binaries.
+development or CI checks for differences between the latest release and the artefact in use.
 
 See [artifacts.md](artifacts.md) about `artifacts.go`.
 
@@ -14,9 +14,9 @@ Usage
 $ generate-artifacts [OPTIONS]
 ```
 
-| Option      | Default value | Description                         |
-| ----------- | ------------- | ----------------------------------- |
-| `--release` | false         | Generate for `artifacts_release.go` |
+| Option      | Default value | Description                      |
+| ----------- | ------------- | -------------------------------- |
+| `--release` | false         | Generate for `artifacts_prod.go` |
 
 Environment variables
 ---------------------