Add performance profiling pipeline to perf tests

.ci/perf_tests/dockerfiles/perf_tests.Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.11
 
 FROM mcr.microsoft.com/azure-functions/python:4-python$PYTHON_VERSION
 
@@ -15,7 +15,8 @@ ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
     FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED=1
 
 RUN apt-get --quiet update && \
-    apt-get install --quiet -y git && \
+    apt-get install --quiet -y git procps && \
+    # Procps is required for displaying worker and profiling processes info
     cd /home && \
     git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \
     mkdir -p /home/site/wwwroot/ && \

.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile

@@ -194,7 +194,8 @@ VOLUME ["/azure-functions-host/workers/python/3.8/LINUX/X64/azure_functions_work
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 && \
     echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list  && \
     apt-get update && \
-    apt-get install -y git k6 && \
+    apt-get install -y git k6 procps && \
+    # Procps is required for displaying worker and profiling processes info
     cd /home && \
     git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \
     mkdir -p /home/site/wwwroot/ && \

.ci/perf_tests/k6scripts/SyncHttpTriggerHelloWorld.js → .ci/perf_tests/k6scripts/SyncHelloWorld.js

@@ -35,7 +35,7 @@ export let options = {
 
 // Main function
 export default function () {
-    let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHttpTriggerHelloWorld`);
+    let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHelloWorld`);
 
     // check() returns false if any of the specified conditions fail
     let checkRes = check(response, {
@@ -45,4 +45,4 @@ export default function () {
 
     // We reverse the check() result since we want to count the failures
     failureRate.add(!checkRes);
-}
+}

.github/workflows/perf-testing-setup.yml

@@ -2,35 +2,37 @@ name: Throughput testing workflow
 
 on:
   workflow_dispatch:
-  pull_request:
-    branches: [ dev ]
-
+    inputs:
+      profiling_sampling_rate:
+        description: 'Profiling sampling rate (tps)'
+        required: false
+        default: '500'
+      test_to_run:
+        description: 'List of perf tests to run'
+        required: false
+        default: SyncHelloWorld
+  issue_comment:                                     
+    types: [created]
 env:
   TESTS_DIR_PATH: ".ci/perf_tests/k6scripts/"
-  PYTHON_VERSION: "3.8"
+  PYTHON_VERSION: "3.10"
+  PYTHON_FUNCTION_PROFILING_STORAGE_ACCT: "azpyfuncpipelinestorage"
   PORT: 8000
 
 jobs:
   build:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.issue.pull_request && contains(github.event.comment.body, '/profile') }} 
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        test_to_run: [ SyncHttpTriggerHelloWorld ]
+        test_to_run: ['${{ github.event.inputs.test_to_run }}']
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ env.PYTHON_VERSION }}
-    - name: Set up Dotnet 2.x
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: '3.1.405'
-    - name: Set up Dotnet 6.x
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: '6.x'
     - name: Setup k6 for throughput testing
       run: |
         cd $GITHUB_WORKSPACE
@@ -47,13 +49,76 @@ jobs:
         python setup.py extension
     - name: Build and Run the Docker image
       run: |
+        echo "Building Docker image with Python version ${{ env.PYTHON_VERSION }}"
         docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --file .ci/perf_tests/dockerfiles/perf_tests.Dockerfile --tag perfimage:latest .
-        docker run -d --shm-size="2g" --env FUNCTIONS_WORKER_RUNTIME_VERSION=${{ env.PYTHON_VERSION }} -p ${PORT}:80 -v $GITHUB_WORKSPACE/azure_functions_worker:/azure-functions-host/workers/python/${{ env.PYTHON_VERSION }}/LINUX/X64/azure_functions_worker perfimage:latest
+
+        echo "Running Docker container..."
+        container_id=$(docker run -d --privileged --env FUNCTIONS_WORKER_RUNTIME_VERSION=${{ env.PYTHON_VERSION }} -p ${PORT}:80 -v $GITHUB_WORKSPACE/azure_functions_worker:/azure-functions-host/workers/python/${{ env.PYTHON_VERSION }}/LINUX/X64/azure_functions_worker perfimage:latest)
         sleep 10  # host needs some time to start.
+        echo "Container ID is $container_id"
+
+        echo "Fetching Docker container logs..."
+        docker logs $container_id
+        worker_pid=$(docker exec $container_id sh -c "ps aux | grep '[p]ython'" | awk '{print $2}')
+        echo "Python worker process id is $worker_pid"
+
+        echo "container_id=$container_id" >> $GITHUB_ENV
+        echo "worker_pid=$worker_pid" >> $GITHUB_ENV
+
     - name: Validate if the functions are now running
       run: |
         curl --get http://localhost:${PORT}/api/${{ matrix.test_to_run }}
+
+    - name: Start py-spy in the background
+      run: |
+        docker exec $container_id sh -c "pip install py-spy"
+        docker exec $container_id sh -c "mkdir /home/profiling_reports"
+        profiling_sampling_rate=${{ github.event.inputs.profiling_sampling_rate }}
+        # report_name="${GITHUB_REF#refs/heads/}_${{ github.run_id }}.svg"
+        report_name="${{ github.run_id }}.svg"
+        docker exec -d $container_id sh -c "RUST_BACKTRACE=1 py-spy record -p $worker_pid -o /home/profiling_reports/$report_name -f flamegraph --idle --nonblocking --rate $profiling_sampling_rate > /home/site/wwwroot/py-spy.log 2>&1 &"
+        sleep 2  # Give it a moment to start
+        py_spy_id=$(docker exec $container_id sh -c "ps aux | grep '[p]y-spy record'" | awk '{print $2}')
+
+        echo "py_spy_id=$py_spy_id" >> $GITHUB_ENV
+        echo "report_name=$report_name" >> $GITHUB_ENV
+
     - name: Run Throughput tests
       run: |
         chmod 755 .ci/perf_tests/run-perftests.sh
         .ci/perf_tests/run-perftests.sh localhost $PORT ${{ env.TESTS_DIR_PATH }} ${{ matrix.test_to_run }}
+
+    - name: Stop profiling and generate report
+      run: |
+        echo "Tests completed, terminating py-spy..."
+        docker exec $container_id cat /home/site/wwwroot/py-spy.log
+        docker exec $container_id sh -c "kill -2 $py_spy_id"
+        sleep 2
+
+        mkdir profiling_reports
+        chmod 777 profiling_reports
+        docker cp $container_id:/home/profiling_reports/$report_name profiling_reports
+
+    - name: Upload SVG to Azure Blob Storage
+      uses: bacongobbler/[email protected]
+      with:
+        source_dir: 'profiling_reports'  # Directory containing the $report_name file
+        container_name: 'profiling'
+        connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
+        sync: 'false'
+
+    - name: Output Blob URL
+      run: |
+        blob_url="https://${{ env.PYTHON_FUNCTION_PROFILING_STORAGE_ACCT }}.blob.core.windows.net/profiling/${{ env.report_name }}"
+        echo "You can view the Blob at: $blob_url"
+
+    - name: Upload profiling result to artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: py-spy-output
+        path: 'profiling_reports/${{ env.report_name }}'
+
+    - name: Create Artifact Link
+      run: |
+        echo "You can download the SVG artifact from the Actions run page."
+        echo "Link to the Actions run page: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"