Fuzz Testing Implementation

.github/workflows/fuzz-test.yml

@@ -0,0 +1,71 @@
+name: Runtime Fuzz Testing
+
+on:
+  workflow_dispatch:
+    inputs:
+      num_of_iterations:
+        description: Number of iterations per fuzzable variable
+        required: False
+        default: 5
+
+jobs:
+  build:
+    runs-on:
+      - self-hosted
+      - linux
+      - x64
+      - container
+
+    container:
+      image: ghcr.io/intel/fpga-runtime-for-opencl/ubuntu-22.04-dev:main
+
+    name: Fuzz Testing
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Clone Radamsa
+      run: |
+        mkdir radamsa_repo
+        cd radamsa_repo
+        git clone https://gitlab.com/akihe/radamsa.git .
+    - name: Install Radamsa
+      run: |
+        cd radamsa_repo
+        make
+        sudo make install
+        cd ..
+    - name: Install PyYAML 
+      run: pip install pyyaml
+    - name: Checkout runtime
+      uses: actions/checkout@v3
+    - name: Build
+      run: | 
+        mkdir -p build/fuzz_testing
+        cd build/fuzz_testing
+        CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
+        ninja -v
+    - name: Fuzz testing
+      run: |
+        cd build/fuzz_testing
+        ls
+        cd fuzz_testing/script
+        export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
+        NUM_OF_ITERATIONS=${{ github.event.inputs.num_of_iterations }}
+        # This if block is only used during testing, because if this workflow is triggered via pull_request, ${{ github.event.inputs.num_of_iterations }} would be empty
+        if [ -z "${NUM_OF_ITERATIONS}" ]; then
+          NUM_OF_ITERATIONS=1
+        fi
+        python3 fuzz_test.py --all -n $NUM_OF_ITERATIONS
+    - name: Peek results
+      run: |
+        cat build/fuzz_testing/fuzz_testing/results/results.yml
+    - name: Upload results
+      uses: actions/upload-artifact@v3
+      with:
+        name: fpga-runtime-for-opencl-${{ github.sha }}-fuzz-test-results-${{ github.run_id }}
+        path: |
+          /__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/results/results.yml
+          /__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/test_outputs
+        if-no-files-found: error

CMakeLists.txt

@@ -404,3 +404,10 @@ install(FILES
 
 add_subdirectory(lib)
 add_subdirectory(test)
+
+# Flag for building fuzz tests
+option(FUZZ_TESTING "Build fuzz tests" OFF)
+message(STATUS "Build fuzz tests: ${FUZZ_TESTING}")
+if(FUZZ_TESTING)
+  add_subdirectory(fuzz_testing)
+endif()

fuzz_testing/CMakeLists.txt

@@ -0,0 +1,6 @@
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+
+add_subdirectory(test)
+file(COPY original_inputs DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY script DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

fuzz_testing/README.md

@@ -0,0 +1,42 @@
+# Fuzz Testing
+
+## Context
+"In programming and software development, fuzzing or fuzz testing is an automated software testing technique that involves providing invalid, unexpected, or random data as inputs to a computer program. The program is then monitored for exceptions such as crashes, failing built-in code assertions, or potential memory leaks."
+
+## How to do fuzz testing on Github 
+1. Click on the `Actions` tab
+2. Click on `Runtime Fuzz Testing` workflow
+3. Click on `Run workflow` (Dropdown button)
+4. Enter the number of iterations you want a single variable to be mutated
+5. Click on `Run workflow` (Green button)
+6. Wait until the test finishes
+7. Inside the workflow run, click `Peek results` to look at the results
+8. Download the artifact from the `Job Summary` to view the full output & result
+
+## How to do fuzz testing locally 
+1. Follow [Prerequisites](https://github.com/intel/fpga-runtime-for-opencl#prerequisites) & [Build](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) & [Test](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) instructions to get a working build. Make sure that the unit tests finishes successfully.
+2. Install Radamsa 0.6+, Python 3.10.7+ and Pyyaml/6.0+
+3. Run the following at the top level in the runtime repo
+```
+mkdir -p build/fuzz_testing
+cd build/fuzz_testing
+CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
+ninja -v
+cd fuzz_testing/script
+export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
+python3 fuzz_test.py --all
+```
+4. A results directory and a test_output directory will be created after the fuzz test finishes
+
+## Classifying Failures
+- Address Sanitizer Error (ASAN Errors): Any errors caught by address sanitizer such as memory leaks (Not acceptable)
+- Aborted Runs: The test aborted/crashed during execution (Not acceptable)
+- Assertion Failures: The test failed due to an assertion (Acceptable)
+- Failed Runs: The test failed because the CHECK statements failed (Acceptable)
+- Hangs: The program did not terminate (Not acceptable)
+- Successful runs: The test succeeded (Acceptable)
+- Test errors: The test failed due to bugs in the fuzz testing infrastructure (Acceptable)
+
+## Additional Notes
+- Currently we are doing most of the fuzz testing on auto discovery strings because these strings are indirectly given by the user. 
+- The fuzz tests are initialized based on the current unit tests, however they should be regarded as separate tests. (i.e. You can make fuzz_test/unit_test only changes)

fuzz_testing/fuzz_src/fuzz_testing.h

@@ -0,0 +1,195 @@
+/*
+  This header file contains all the functions that are needed for fuzz testing
+*/
+
+#ifndef FUZZ_TESTING_H
+#define FUZZ_TESTING_H
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+using namespace std;
+
+#define YML ".yml"
+#define TAB "  "
+#define TAB_LENGTH_FOR_VAR 4
+
+#define FUZZ_TEST_ERROR "Fuzz test error: "
+
+// groupName-testname-varName => data
+inline unordered_map<string, string> data_map;
+
+// Basically find terminating '"' that concludes the variable value
+inline bool parseVariableValue(ifstream &inputFile, string &returnVal) {
+  string line;
+  while (getline(inputFile, line)) {
+    returnVal = returnVal + "\n" + line;
+    // Terminating '"' found
+    if (line[line.length() - 1] == '"') {
+      // Remove double quotes
+      returnVal = returnVal.substr(1, returnVal.length() - 2);
+      return true;
+    }
+  }
+  // EOF
+  return false;
+}
+
+// Generate a string that combines the groupName testName and varName (i.e.
+// groupName-testname-varName)
+inline string generate_key(const vector<string> &names) {
+  if (names.size() != 3) {
+    cout << FUZZ_TEST_ERROR << "Incorrect key size" << endl;
+    exit(1);
+  }
+  return names[0] + "--" + names[1] + "--" + names[2];
+}
+
+// A utility function that counts how many "TAB"s are in the current line
+// A line should only have at most 2 tabs in the current schema
+inline int countTabs(const string &line) {
+  int count = 0;
+  for (unsigned int i = 0; i < line.size(); i++) {
+    if (line[i] == ' ') {
+      count++;
+    } else {
+      if (count % 2 != 0) {
+        cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
+             << endl;
+        exit(1);
+      }
+      int indent = count / 2;
+      if (indent > 2) {
+        cout << FUZZ_TEST_ERROR << "Yaml has too much indentation " << endl;
+        exit(1);
+      }
+      return indent;
+    }
+  }
+  // A line of spaces, yaml error
+  cout << FUZZ_TEST_ERROR << "Yaml should not have an empty line " << endl;
+  exit(1);
+}
+
+// A function that loads fuzzed data from a input yaml file to data_map
+inline void preload(ifstream &inputFile) {
+  string line;
+  vector<string> names;
+  // 0 => group, 1 => test, 2 => variable
+  int lastIndent = 0;
+  bool first = true;
+
+  while (getline(inputFile, line)) {
+    // Parsing yaml file manually
+    int currentIndent = countTabs(line);
+    if (currentIndent - 1 > lastIndent) {
+      cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
+           << endl;
+      exit(1);
+    }
+    int diff = lastIndent - currentIndent;
+    // Delete leaf if exiting
+    if (diff >= 0 && !first) {
+      names.resize(names.size() - diff - 1);
+    }
+    stringstream ss_line(line);
+    string curr_name;
+    ss_line >> curr_name;
+    // If current indent level is at group/test
+    if (currentIndent <= 1) {
+      // Remove : at the end and add them to names
+      names.push_back(curr_name.substr(0, curr_name.length() - 1));
+    } else {
+      // Else the current indent level must be at variable
+      stringstream ss(line);
+      string varName;
+      string varValue;
+      // Note: varName contains ':'
+      ss >> varName;
+      names.push_back(varName.substr(0, varName.length() - 1));
+      // Note that varValue contains ""
+      getline(ss, varValue);
+      // Bypass leading space
+      varValue = varValue.substr(1, varValue.size() - 1);
+      string key = generate_key(names);
+      if (varValue[varValue.length() - 1] == '"') {
+        // Filter out double quote characters
+        data_map.insert(pair<string, string>(
+            key, varValue.substr(1, varValue.length() - 2)));
+      } else {
+        // The variable value contains newline character
+        // Needs to parse multiple lines until finding the terminating '"'
+        if (parseVariableValue(inputFile, varValue)) {
+          data_map.insert(pair<string, string>(key, varValue));
+        }
+        // EOF reached but still couldn't find terminating "
+        else {
+          cout << FUZZ_TEST_ERROR
+               << "EOF reached but still couldn't find terminating \"" << endl;
+          exit(1);
+        }
+      }
+    }
+    lastIndent = currentIndent;
+    first = false;
+  }
+}
+
+// A top level function that preloads all fuzzed data into the data_map
+inline void preload_data(string fileName) {
+  cout << "\nPreloading fuzzed data starts: " << endl;
+  string pathPrefix = "../mutated_inputs/";
+  string path = pathPrefix + fileName + YML;
+  cout << "Opening input file: " << path << endl;
+  ifstream inputFile(path.c_str());
+  string value;
+  if (inputFile.is_open()) {
+    preload(inputFile);
+    cout << "Preloading fuzzed data ends " << endl;
+  } else {
+    cout << FUZZ_TEST_ERROR << "Unable to open fuzz test file! " << endl;
+    cout << "Make sure you run the fuzz test in the test directory" << endl;
+    exit(1);
+  }
+}
+
+// Parameters: group name, test name, variable name
+inline string load_fuzzed_value(string groupName, string testName,
+                                string varName) {
+  vector<string> names = {groupName, testName, varName};
+  string key = generate_key(names);
+  if (data_map.find(key) == data_map.end()) {
+    cout << FUZZ_TEST_ERROR << "Unable to find " << key << endl;
+    exit(1);
+  }
+  return data_map[key];
+}
+
+// Cast data to corresponding data types
+template <class castType>
+inline castType load_fuzzed_value_cast(string groupName, string testName,
+                                       string varName) {
+  try {
+    // Convert to unsigned long long then convert to castType
+    string value = load_fuzzed_value(groupName, testName, varName);
+    int base = 10;
+    if (value.size() > 2 && value.substr(0, 2) == "0x") {
+      base = 16;
+    }
+    return (castType)stoull(value, 0, base);
+  } catch (string e) {
+    cout << FUZZ_TEST_ERROR << "Data type mismatch " << endl;
+    cout << e << endl;
+    exit(1);
+  }
+}
+
+// Utility functions
+inline bool check_condition(bool condition, bool &check) {
+  check = condition;
+  return check;
+}
+#endif