From 4e0559ad64879182bdcbb8c1f6f2e9f6d18e68c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Sun, 1 Sep 2024 21:43:10 +0200 Subject: [PATCH 1/5] Add version benchmark to CI --- .github/workflows/benchmark.yml | 111 ++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000000..5ae3da7630cb2 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,111 @@ +name: Benchmark +on: + schedule: + - cron: "0 2 * * *" +permissions: + contents: read +jobs: + BENCHMARK: + name: BENCHMARK + if: github.repository == 'php/php-src' + runs-on: ubuntu-22.04 + steps: + - name: Install dependencies + run: | + set -ex + sudo apt-get update + sudo apt-get install gpg + + wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt update + sudo apt install terraform=1.5.7-* + - name: Checkout benchmark suite + uses: actions/checkout@v4 + with: + repository: 'kocsismate/php-version-benchmarks' + ref: 'main' + fetch-depth: 1 + path: 'php-version-benchmarks' + - name: Checkout php-src + uses: actions/checkout@v4 + with: + repository: 'php/php-src' + ref: 'master' + fetch-depth: 100 + path: 'php-version-benchmarks/tmp/php_master' + - name: Copy-paste the same repo content for benchmarking JIT + run: | + set -e + + cp -r "php-version-benchmarks/tmp/php_master/" "php-version-benchmarks/tmp/php_master_jit" + - name: Setup benchmark config + run: | + set -e + + cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini + ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g') + sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini + + cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + + cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master1.ini + YESTERDAY="$(date -d "-2 day 13:00" '+%Y-%m-%d')" + YESTERDAY_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY 23:59:59" -n 1 --pretty='%H')" + sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master1.ini + sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master1.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=$YESTERDAY_SHA/g" ./php-version-benchmarks/config/php/master1.ini + cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master2.ini + cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master2_jit.ini + + cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini + cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini + cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini + cp ./php-version-benchmarks/config/test/5_bench.php.ini.dist ./php-version-benchmarks/config/test/5_bench.php.ini + cp ./php-version-benchmarks/config/test/6_micro_bench.php.ini.dist ./php-version-benchmarks/config/test/6_micro_bench.php.ini + + rm -rf ./php-version-benchmarks/docs/results + - name: Git setup + run: | + git config --global user.name "Benchmark" + git config --global user.email "benchmark@php.net" + - name: Checkout benchmark results + uses: actions/checkout@v4 + with: + repository: kocsismate/php-version-benchmark-results + ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} + path: 'php-version-benchmarks/docs/results' + - name: Run benchmark + run: ./php-version-benchmarks/benchmark.sh run aws + - name: Store results + run: | + set -ex + + cd ./php-version-benchmarks/docs/results + git pull --autostash + if [ -e ".git/MERGE_HEAD" ]; then + echo "Merging, can't proceed" + exit 1 + fi + git add . + if git diff --cached --quiet; then + exit 0 + fi + git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}" + git push + - name: Cleanup + if: always() + run: | + set -ex + + rm -rf ./php-version-benchmarks/tmp/ + rm -f ./php-version-benchmarks/build/infrastructure/config/*.tfvars + rm -rf ./php-version-benchmarks/build/infrastructure/aws/.terraform/ + rm -f ./php-version-benchmarks/build/infrastructure/aws/.terraform.lock.hcl + rm -f ./php-version-benchmarks/build/infrastructure/aws/aws.tfplan + rm -f ./php-version-benchmarks/build/infrastructure/aws/terraform.tfstate + rm -f ./php-version-benchmarks/build/infrastructure/aws/terraform.tfstate.backup + rm -f ./php-version-benchmarks/config/infra/aws/*.ini From efd87aa5c7b429e8474952b77cf4348ad4de4ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Sat, 7 Sep 2024 13:49:37 +0200 Subject: [PATCH 2/5] Review fixes --- .github/workflows/benchmark.yml | 64 +++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5ae3da7630cb2..2127dbed5ee75 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,7 @@ name: Benchmark on: schedule: - - cron: "0 2 * * *" + - cron: "30 0 * * *" permissions: contents: read jobs: @@ -19,8 +19,9 @@ jobs: wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list - sudo apt update - sudo apt install terraform=1.5.7-* + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update -y + sudo apt-get install -y terraform=1.5.7-* - name: Checkout benchmark suite uses: actions/checkout@v4 with: @@ -32,52 +33,63 @@ jobs: uses: actions/checkout@v4 with: repository: 'php/php-src' - ref: 'master' + ref: '${{ github.sha }}' fetch-depth: 100 path: 'php-version-benchmarks/tmp/php_master' - - name: Copy-paste the same repo content for benchmarking JIT + - name: Setup benchmark results run: | - set -e - - cp -r "php-version-benchmarks/tmp/php_master/" "php-version-benchmarks/tmp/php_master_jit" - - name: Setup benchmark config + git config --global user.name "Benchmark" + git config --global user.email "benchmark@php.net" + + rm -rf ./php-version-benchmarks/docs/results + - name: Checkout benchmark results + uses: actions/checkout@v4 + with: + repository: kocsismate/php-version-benchmark-results + ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} + path: 'php-version-benchmarks/docs/results' + - name: Set benchmark config run: | set -e + # Set infrastructure config cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g') sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini - cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + YEAR="$(date '+%Y')" + DATABASE="./php-version-benchmarks/docs/results/$YEAR/database.tsv" + if [ -f "$DATABASE" ]; then + LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)" + else + YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')" + LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" + fi + + # Set config for the previous PHP version cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master1.ini - YESTERDAY="$(date -d "-2 day 13:00" '+%Y-%m-%d')" - YESTERDAY_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY 23:59:59" -n 1 --pretty='%H')" sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master1.ini sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master1.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=$YESTERDAY_SHA/g" ./php-version-benchmarks/config/php/master1.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master1.ini + + # Set config for the current PHP version cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master2.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master2.ini + + # Set config for current PHP version with JIT + git clone ./php-version-benchmarks/tmp/php_master/ ./php-version-benchmarks/tmp/php_master_jit cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master2_jit.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master2_jit.ini + # Set test configs cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini cp ./php-version-benchmarks/config/test/5_bench.php.ini.dist ./php-version-benchmarks/config/test/5_bench.php.ini cp ./php-version-benchmarks/config/test/6_micro_bench.php.ini.dist ./php-version-benchmarks/config/test/6_micro_bench.php.ini - - rm -rf ./php-version-benchmarks/docs/results - - name: Git setup - run: | - git config --global user.name "Benchmark" - git config --global user.email "benchmark@php.net" - - name: Checkout benchmark results - uses: actions/checkout@v4 - with: - repository: kocsismate/php-version-benchmark-results - ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} - path: 'php-version-benchmarks/docs/results' - name: Run benchmark run: ./php-version-benchmarks/benchmark.sh run aws - name: Store results @@ -92,7 +104,7 @@ jobs: fi git add . if git diff --cached --quiet; then - exit 0 + exit 1 fi git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}" git push From 23fcb24ef89b7557a2c9554c756beb498af4bc95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Thu, 12 Sep 2024 08:55:39 +0200 Subject: [PATCH 3/5] Measure results for a baseline PHP version The commit is now the latest one. This should be changed later for the one after PHP 8.4 is branched. --- .github/workflows/benchmark.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2127dbed5ee75..3789184297eae 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -69,20 +69,29 @@ jobs: LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" fi + BASELINE_SHA="306a51951f460abc3a7ebf4d196474d4c041b87e" + BASELINE_SHORT_SHA="$(echo "$BASELINE_SHA" | cut -c1-4)" + + # Set config for the baseline PHP version + cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_baseline.ini + sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - baseline@'"$BASELINE_SHORT_SHA"'"/g' ./php-version-benchmarks/config/php/master_baseline.ini + sed -i "s/PHP_ID=php_master/PHP_ID=php_master_baseline/g" ./php-version-benchmarks/config/php/master_baseline.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=$BASELINE_SHA/g" ./php-version-benchmarks/config/php/master_baseline.ini + # Set config for the previous PHP version - cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master1.ini - sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master1.ini - sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master1.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master1.ini + cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_last.ini + sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master_last.ini + sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master_last.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master_last.ini # Set config for the current PHP version - cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master2.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master2.ini + cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_now.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now.ini # Set config for current PHP version with JIT git clone ./php-version-benchmarks/tmp/php_master/ ./php-version-benchmarks/tmp/php_master_jit - cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master2_jit.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master2_jit.ini + cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master_now_jit.ini + sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now_jit.ini # Set test configs cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini From 24fcc0cdc19c63ce1a8f284a04dcfbc773e2e763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Tue, 24 Sep 2024 22:21:57 +0200 Subject: [PATCH 4/5] Update baseline commit SHA + result repo name --- .../{benchmark.yml => real-time-benchmark.yml} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{benchmark.yml => real-time-benchmark.yml} (97%) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/real-time-benchmark.yml similarity index 97% rename from .github/workflows/benchmark.yml rename to .github/workflows/real-time-benchmark.yml index 3789184297eae..05c2a081710c9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/real-time-benchmark.yml @@ -1,12 +1,12 @@ -name: Benchmark +name: Real-time Benchmark on: schedule: - cron: "30 0 * * *" permissions: contents: read jobs: - BENCHMARK: - name: BENCHMARK + REAL_TIME_BENCHMARK: + name: REAL_TIME_BENCHMARK if: github.repository == 'php/php-src' runs-on: ubuntu-22.04 steps: @@ -45,7 +45,7 @@ jobs: - name: Checkout benchmark results uses: actions/checkout@v4 with: - repository: kocsismate/php-version-benchmark-results + repository: php/php-version-benchmark-results ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} path: 'php-version-benchmarks/docs/results' - name: Set benchmark config @@ -69,7 +69,7 @@ jobs: LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" fi - BASELINE_SHA="306a51951f460abc3a7ebf4d196474d4c041b87e" + BASELINE_SHA="d5f6e56610c729710073350af318c4ea1b292cfe" BASELINE_SHORT_SHA="$(echo "$BASELINE_SHA" | cut -c1-4)" # Set config for the baseline PHP version From dd0cd0ee3859f71ec014f4fe96058b1e2ab023eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Wed, 25 Sep 2024 14:48:43 +0200 Subject: [PATCH 5/5] Rename benchmark results repo after transfer --- .github/workflows/real-time-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/real-time-benchmark.yml b/.github/workflows/real-time-benchmark.yml index 05c2a081710c9..9e1fa9fdbe6a2 100644 --- a/.github/workflows/real-time-benchmark.yml +++ b/.github/workflows/real-time-benchmark.yml @@ -42,10 +42,10 @@ jobs: git config --global user.email "benchmark@php.net" rm -rf ./php-version-benchmarks/docs/results - - name: Checkout benchmark results + - name: Checkout benchmark data uses: actions/checkout@v4 with: - repository: php/php-version-benchmark-results + repository: php/real-time-benchmark-data ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} path: 'php-version-benchmarks/docs/results' - name: Set benchmark config