From 28c506d3412178e37b8aa3829baa28c535306e0a Mon Sep 17 00:00:00 2001
From: BD103 <59022059+BD103@users.noreply.github.com>
Date: Sat, 8 Jun 2024 23:07:05 -0400
Subject: [PATCH] New benchmarks (#1)

* feat: create custom benchmarks

* feat: update github actions for new benchmarks

* fix: checkout repository first

* fix: don't optimize out `world.get_entity`

* chore: add comments

* fix: always use latest bencher version

* feat: update bencher command

* chore: final comments
---
 .github/workflows/daily.yml | 24 +++++++--------
 .gitignore                  |  6 ++++
 Cargo.toml                  | 18 +++++++++++
 benches/ecs/mod.rs          | 61 +++++++++++++++++++++++++++++++++++++
 benches/ecs/types.rs        | 13 ++++++++
 benches/main.rs             | 13 ++++++++
 src/lib.rs                  |  1 +
 7 files changed, 123 insertions(+), 13 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 Cargo.toml
 create mode 100644 benches/ecs/mod.rs
 create mode 100644 benches/ecs/types.rs
 create mode 100644 benches/main.rs
 create mode 100644 src/lib.rs

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index d734b20..0ca847e 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -16,38 +16,36 @@ env:
 
 jobs:
   bench:
-    name: Bench
+    name: Run benchmarks
     runs-on: ubuntu-latest
-    env:
-      BENCHER_PROJECT: bevy
     steps:
-      - name: Checkout Bevy
+      - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          repository: bevyengine/bevy
 
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache build files
         uses: Leafwing-Studios/cargo-cache@v1
-        with:
-          cargo-target-dir: benches/target
 
       - name: Install Bencher
-        uses: bencherdev/bencher@v0.4.10
+        uses: bencherdev/bencher@main
 
+      # Run benchmarks, piping output to both `results.txt` and stdout.
       - name: Run benchmarks
-        working-directory: benches
         run: cargo bench -- 2>&1 | tee results.txt
 
       - name: Upload benchmarks
         env:
+          BENCHER_PROJECT: bevy
           BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+          BENCHER_TESTBED: github-actions
+        # --no-hash: Do not use the Git hash, because we care about the Bevy commit and not our
+        #     own.
+        # --file: Upload saved Criterion output instead of running a command.
         run: |
           bencher run \
-            --branch main \
-            --testbed github-actions \
             --adapter rust_criterion \
+            --no-hash \
             --err \
-            --file benches/results.txt
+            --file results.txt
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ad8c012
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+# Rust
+/target
+Cargo.lock
+
+# Used for local testing
+.env
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..4541650
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "bevy-bencher"
+edition = "2021"
+publish = false
+
+[dev-dependencies]
+# Benchmarking library.
+criterion = "0.5.1"
+
+# Seedable PRNG
+rand = { version = "0.8.5", default-features = false, features = ["small_rng"] }
+
+# Bevy, on the latest commit.
+bevy_ecs = { git = "https://github.com/bevyengine/bevy.git" }
+
+[[bench]]
+name = "main"
+harness = false
diff --git a/benches/ecs/mod.rs b/benches/ecs/mod.rs
new file mode 100644
index 0000000..2bc57bf
--- /dev/null
+++ b/benches/ecs/mod.rs
@@ -0,0 +1,61 @@
+mod types;
+
+use self::types::*;
+use bevy_ecs::prelude::*;
+use criterion::{criterion_group, BatchSize, Criterion};
+use rand::prelude::*;
+use std::{hint::black_box, iter::repeat};
+
+/// Benchmarks spawning an entity into a [`World`].
+pub fn world_spawn(c: &mut Criterion) {
+    c.bench_function("world_spawn", |b| {
+        let mut world = World::new();
+
+        b.iter(|| {
+            world.spawn((A(0), B(0)));
+        });
+    });
+}
+
+/// Benchmarks iterating over all matching entities within a [`World`].
+pub fn world_query_iter(c: &mut Criterion) {
+    c.bench_function("world_query_iter", |b| {
+        let mut world = World::new();
+
+        // Spawn some with `A`, some with `A` and `B`, and some without `A`.
+        world.spawn_batch(repeat((A(0), B(0))).take(64));
+        world.spawn_batch(repeat(A(0)).take(64));
+        world.spawn_batch(repeat(B(0)).take(64));
+
+        b.iter(|| {
+            for a in world.query::<&A>().iter(&world) {
+                // Pretend we're doing something with `a` so this is not optimized away.
+                black_box(a);
+            }
+        });
+    });
+}
+
+pub fn world_get_entity(c: &mut Criterion) {
+    c.bench_function("world_get_entity", |b| {
+        let mut world = World::new();
+        let mut prng = crate::create_prng();
+
+        // Spawn 64 entities and store their IDs in a list.
+        let ids: Vec<Entity> = world.spawn_batch(repeat((A(0), B(0))).take(64)).collect();
+
+        b.iter_batched(
+            // Pick a random ID from the list, outside of the benchmark.
+            || ids.choose(&mut prng).unwrap(),
+            |&input| {
+                // Get the entity from the world. The input is random to hopefully escape any
+                // locality optimizations. We use `.get_entity()` instead of `.entity()` to avoid
+                // an extra `.unwrap()` call, though the result will never be `None`.
+                black_box(world.get_entity(input));
+            },
+            BatchSize::SmallInput,
+        );
+    });
+}
+
+criterion_group!(group, world_spawn, world_query_iter, world_get_entity);
diff --git a/benches/ecs/types.rs b/benches/ecs/types.rs
new file mode 100644
index 0000000..93a9974
--- /dev/null
+++ b/benches/ecs/types.rs
@@ -0,0 +1,13 @@
+use bevy_ecs::prelude::*;
+
+/// A 32-bit wide component.
+///
+/// This is intended to not match with [`B`]'s size, to force the ECS to deal with padding.
+#[derive(Component, Clone)]
+pub struct A(pub u32);
+
+/// A 16-bit wide component.
+///
+/// This is intended to not match with [`A`]'s size, to force the ECS to deal with padding.
+#[derive(Component, Clone)]
+pub struct B(pub u16);
diff --git a/benches/main.rs b/benches/main.rs
new file mode 100644
index 0000000..c224a3a
--- /dev/null
+++ b/benches/main.rs
@@ -0,0 +1,13 @@
+mod ecs;
+
+use criterion::criterion_main;
+use rand::{prelude::*, rngs::SmallRng};
+
+pub(crate) fn create_prng() -> impl Rng {
+    // A small and fast psuedo-random number generator. This is not reproducible across 32-bit
+    // and 64-bit platforms, but that shouldn't be a problem here. The seed was chosen somewhat
+    // randomly, but modified to have a nice distribution of 1s and 0s.
+    SmallRng::seed_from_u64(0x7df09deb486e920a)
+}
+
+criterion_main!(ecs::group);
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 0000000..dd2e0b5
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1 @@
+// Nothing here, check out the benches folder!