Skip to content

Commit f2d041a

Browse files
author
Tyler Zhao
committed
Fuzz Testing Implementation
Add support for running multiple fuzz tests Add support for loading different data type fuzzed data Remove files Add fuzz variable Correct header path Bugfixes Generate Coverage Report after Fuzz test Add inputs for acl_auto_configure_test Update to be more user friendly Workaround to bypass unit test memory leak on local variables Workaround for unit test memory leaks Add the result yaml file Add assertion check Workflow added Remove shelling Rename fuzz testing directory and configured cmake with FUZZ_TEST option Clang format Test workflow Add container Install radamsa in workflow Syntax fix Modify docker file Change pyyaml version Test Remove : Test Test Add sudo Test Test Test Test Test Test Test Test Python3 fix Check python3 instead Python3! Bugfix Bugfix Add pyyaml Install Pyyaml Bugfix Update fuzz_testing.py Remove generate coverage report when running from workflow Bugfix Bugfix Upload outputs Debug Bugfix Bugfix Bugfix Install lcov Intentional ASAN leak Bugfix catch ASAN error Intentional leak Peek results in workflow Check timeout error Check assertion failure Remove intentional assertion Remove untested unit test files Documentation Update README.md Update README.md Remove untested files Use github machine Install pyyaml correctly Delete BSP in fuzz testing folder Edit readme Remove fake_bsp Bugfix Bugfix Revert to using self hosted machine More comments added Documentation Clang format Name change Remove comments and clang format
1 parent 17d71d1 commit f2d041a

16 files changed

+4544
-0
lines changed

.github/workflows/fuzz-test.yml

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Runtime Fuzz Testing
2+
3+
on:
4+
# For the purpose of testing the workflow
5+
pull_request:
6+
workflow_dispatch:
7+
inputs:
8+
num_of_iterations:
9+
description: Number of iterations per fuzzable variable
10+
required: False
11+
default: 5
12+
13+
jobs:
14+
build:
15+
runs-on:
16+
- self-hosted
17+
- linux
18+
- x64
19+
- container
20+
21+
container:
22+
image: ghcr.io/intel/fpga-runtime-for-opencl/ubuntu-22.04-dev:main
23+
24+
name: Fuzz Testing
25+
steps:
26+
- name: Set up Python
27+
uses: actions/setup-python@v4
28+
with:
29+
python-version: '3.10'
30+
- name: Clone Radamsa
31+
run: |
32+
mkdir radamsa_repo
33+
cd radamsa_repo
34+
git clone https://gitlab.com/akihe/radamsa.git .
35+
- name: Install Radamsa
36+
run: |
37+
cd radamsa_repo
38+
make
39+
sudo make install
40+
cd ..
41+
- name: Install PyYAML
42+
run: pip install pyyaml
43+
- name: Checkout runtime
44+
uses: actions/checkout@v3
45+
- name: Build
46+
run: |
47+
mkdir -p build/fuzz_testing
48+
cd build/fuzz_testing
49+
CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
50+
ninja -v
51+
- name: Fuzz testing
52+
run: |
53+
cd build/fuzz_testing
54+
ls
55+
cd fuzz_testing/script
56+
export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
57+
NUM_OF_ITERATIONS=${{ github.event.inputs.num_of_iterations }}
58+
# This if block is only used during testing, because if this workflow is triggered via pull_request, ${{ github.event.inputs.num_of_iterations }} would be empty
59+
if [ -z "${NUM_OF_ITERATIONS}" ]; then
60+
NUM_OF_ITERATIONS=1
61+
fi
62+
python3 fuzz_test.py --all -n $NUM_OF_ITERATIONS
63+
- name: Peek results
64+
run: |
65+
cat build/fuzz_testing/fuzz_testing/results/results.yml
66+
- name: Upload results
67+
uses: actions/upload-artifact@v3
68+
with:
69+
name: fpga-runtime-for-opencl-${{ github.sha }}-fuzz-test-results-${{ github.run_id }}
70+
path: |
71+
/__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/results/results.yml
72+
/__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/test_outputs
73+
if-no-files-found: error

CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,3 +404,10 @@ install(FILES
404404

405405
add_subdirectory(lib)
406406
add_subdirectory(test)
407+
408+
# Flag for building fuzz tests
409+
option(FUZZ_TESTING "Build fuzz tests" OFF)
410+
message(STATUS "Build fuzz tests: ${FUZZ_TESTING}")
411+
if(FUZZ_TESTING)
412+
add_subdirectory(fuzz_testing)
413+
endif()

fuzz_testing/CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (C) 2021 Intel Corporation
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
4+
add_subdirectory(test)
5+
file(COPY original_inputs DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
6+
file(COPY script DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

fuzz_testing/README.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Fuzz Testing
2+
3+
## Context
4+
"In programming and software development, fuzzing or fuzz testing is an automated software testing technique that involves providing invalid, unexpected, or random data as inputs to a computer program. The program is then monitored for exceptions such as crashes, failing built-in code assertions, or potential memory leaks."
5+
6+
## How to do fuzz testing on Github
7+
1. Click on the `Actions` tab
8+
2. Click on `Runtime Fuzz Testing` workflow
9+
3. Click on `Run workflow` (Dropdown button)
10+
4. Enter the number of iterations you want a single variable to be mutated
11+
5. Click on `Run workflow` (Green button)
12+
6. Wait until the test finishes
13+
7. Inside the workflow run, click `Peek results` to look at the results
14+
8. Download the artifact from the `Job Summary` to view the full output & result
15+
16+
## How to do fuzz testing locally
17+
1. Follow [Prerequisites](https://github.com/intel/fpga-runtime-for-opencl#prerequisites) & [Build](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) & [Test](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) instructions to get a working build. Make sure that the unit tests finishes successfully.
18+
2. Install Radamsa 0.6+, Python 3.10.7+ and Pyyaml/6.0+
19+
3. Run the following at the top level in the runtime repo
20+
```
21+
mkdir -p build/fuzz_testing
22+
cd build/fuzz_testing
23+
CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
24+
ninja -v
25+
cd fuzz_testing/script
26+
export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
27+
python3 fuzz_test.py --all
28+
```
29+
4. A results directory and a test_output directory will be created after the fuzz test finishes
30+
31+
## Classifying Failures
32+
- Address Sanitizer Error (ASAN Errors): Any errors caught by address sanitizer such as memory leaks (Not acceptable)
33+
- Aborted Runs: The test aborted/crashed during execution (Not acceptable)
34+
- Assertion Failures: The test failed due to an assertion (Acceptable)
35+
- Failed Runs: The test failed because the CHECK statements failed (Acceptable)
36+
- Hangs: The program did not terminate (Not acceptable)
37+
- Successful runs: The test succeeded (Acceptable)
38+
- Test errors: The test failed due to bugs in the fuzz testing infrastructure (Acceptable)
39+
40+
## Additional Notes
41+
- Currently we are doing most of the fuzz testing on auto discovery strings because these strings are indirectly given by the user.
42+
- The fuzz tests are initialized based on the current unit tests, however they should be regarded as separate tests. (i.e. You can make fuzz_test/unit_test only changes)

fuzz_testing/fuzz_src/fuzz_testing.h

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
/*
2+
This header file contains all the functions that are needed for fuzz testing
3+
*/
4+
5+
#ifndef FUZZ_TESTING_H
6+
#define FUZZ_TESTING_H
7+
#include <fstream>
8+
#include <iostream>
9+
#include <sstream>
10+
#include <string>
11+
#include <unordered_map>
12+
#include <vector>
13+
14+
using namespace std;
15+
16+
#define YML ".yml"
17+
#define TAB " "
18+
#define TAB_LENGTH_FOR_VAR 4
19+
20+
#define FUZZ_TEST_ERROR "Fuzz test error: "
21+
22+
// groupName-testname-varName => data
23+
inline unordered_map<string, string> data_map;
24+
25+
// Basically find terminating '"' that concludes the variable value
26+
inline bool parseVariableValue(ifstream &inputFile, string &returnVal) {
27+
string line;
28+
while (getline(inputFile, line)) {
29+
returnVal = returnVal + "\n" + line;
30+
// Terminating '"' found
31+
if (line[line.length() - 1] == '"') {
32+
// Remove double quotes
33+
returnVal = returnVal.substr(1, returnVal.length() - 2);
34+
return true;
35+
}
36+
}
37+
// EOF
38+
return false;
39+
}
40+
41+
// Generate a string that combines the groupName testName and varName (i.e.
42+
// groupName-testname-varName)
43+
inline string generate_key(const vector<string> &names) {
44+
if (names.size() != 3) {
45+
cout << FUZZ_TEST_ERROR << "Incorrect key size" << endl;
46+
exit(1);
47+
}
48+
return names[0] + "--" + names[1] + "--" + names[2];
49+
}
50+
51+
// A utility function that counts how many "TAB"s are in the current line
52+
// A line should only have at most 2 tabs in the current schema
53+
inline int countTabs(const string &line) {
54+
int count = 0;
55+
for (unsigned int i = 0; i < line.size(); i++) {
56+
if (line[i] == ' ') {
57+
count++;
58+
} else {
59+
if (count % 2 != 0) {
60+
cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
61+
<< endl;
62+
exit(1);
63+
}
64+
int indent = count / 2;
65+
if (indent > 2) {
66+
cout << FUZZ_TEST_ERROR << "Yaml has too much indentation " << endl;
67+
exit(1);
68+
}
69+
return indent;
70+
}
71+
}
72+
// A line of spaces, yaml error
73+
cout << FUZZ_TEST_ERROR << "Yaml should not have an empty line " << endl;
74+
exit(1);
75+
}
76+
77+
// A function that loads fuzzed data from a input yaml file to data_map
78+
inline void preload(ifstream &inputFile) {
79+
string line;
80+
vector<string> names;
81+
// 0 => group, 1 => test, 2 => variable
82+
int lastIndent = 0;
83+
bool first = true;
84+
85+
while (getline(inputFile, line)) {
86+
// Parsing yaml file manually
87+
int currentIndent = countTabs(line);
88+
if (currentIndent - 1 > lastIndent) {
89+
cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
90+
<< endl;
91+
exit(1);
92+
}
93+
int diff = lastIndent - currentIndent;
94+
// Delete leaf if exiting
95+
if (diff >= 0 && !first) {
96+
names.resize(names.size() - diff - 1);
97+
}
98+
stringstream ss_line(line);
99+
string curr_name;
100+
ss_line >> curr_name;
101+
// If current indent level is at group/test
102+
if (currentIndent <= 1) {
103+
// Remove : at the end and add them to names
104+
names.push_back(curr_name.substr(0, curr_name.length() - 1));
105+
} else {
106+
// Else the current indent level must be at variable
107+
stringstream ss(line);
108+
string varName;
109+
string varValue;
110+
// Note: varName contains ':'
111+
ss >> varName;
112+
names.push_back(varName.substr(0, varName.length() - 1));
113+
// Note that varValue contains ""
114+
getline(ss, varValue);
115+
// Bypass leading space
116+
varValue = varValue.substr(1, varValue.size() - 1);
117+
string key = generate_key(names);
118+
if (varValue[varValue.length() - 1] == '"') {
119+
// Filter out double quote characters
120+
data_map.insert(pair<string, string>(
121+
key, varValue.substr(1, varValue.length() - 2)));
122+
} else {
123+
// The variable value contains newline character
124+
// Needs to parse multiple lines until finding the terminating '"'
125+
if (parseVariableValue(inputFile, varValue)) {
126+
data_map.insert(pair<string, string>(key, varValue));
127+
}
128+
// EOF reached but still couldn't find terminating "
129+
else {
130+
cout << FUZZ_TEST_ERROR
131+
<< "EOF reached but still couldn't find terminating \"" << endl;
132+
exit(1);
133+
}
134+
}
135+
}
136+
lastIndent = currentIndent;
137+
first = false;
138+
}
139+
}
140+
141+
// A top level function that preloads all fuzzed data into the data_map
142+
inline void preload_data(string fileName) {
143+
cout << "\nPreloading fuzzed data starts: " << endl;
144+
string pathPrefix = "../mutated_inputs/";
145+
string path = pathPrefix + fileName + YML;
146+
cout << "Opening input file: " << path << endl;
147+
ifstream inputFile(path.c_str());
148+
string value;
149+
if (inputFile.is_open()) {
150+
preload(inputFile);
151+
cout << "Preloading fuzzed data ends " << endl;
152+
} else {
153+
cout << FUZZ_TEST_ERROR << "Unable to open fuzz test file! " << endl;
154+
cout << "Make sure you run the fuzz test in the test directory" << endl;
155+
exit(1);
156+
}
157+
}
158+
159+
// Parameters: group name, test name, variable name
160+
inline string load_fuzzed_value(string groupName, string testName,
161+
string varName) {
162+
vector<string> names = {groupName, testName, varName};
163+
string key = generate_key(names);
164+
if (data_map.find(key) == data_map.end()) {
165+
cout << FUZZ_TEST_ERROR << "Unable to find " << key << endl;
166+
exit(1);
167+
}
168+
return data_map[key];
169+
}
170+
171+
// Cast data to corresponding data types
172+
template <class castType>
173+
inline castType load_fuzzed_value_cast(string groupName, string testName,
174+
string varName) {
175+
try {
176+
// Convert to unsigned long long then convert to castType
177+
string value = load_fuzzed_value(groupName, testName, varName);
178+
int base = 10;
179+
if (value.size() > 2 && value.substr(0, 2) == "0x") {
180+
base = 16;
181+
}
182+
return (castType)stoull(value, 0, base);
183+
} catch (string e) {
184+
cout << FUZZ_TEST_ERROR << "Data type mismatch " << endl;
185+
cout << e << endl;
186+
exit(1);
187+
}
188+
}
189+
190+
// Utility functions
191+
inline bool check_condition(bool condition, bool &check) {
192+
check = condition;
193+
return check;
194+
}
195+
#endif

0 commit comments

Comments
 (0)