Skip to content

Commit 98ab300

Browse files
Tyler Zhaozibaiwan
authored andcommitted
Fuzz Testing Implementation
1 parent a22ae0c commit 98ab300

16 files changed

+4542
-0
lines changed

.github/workflows/fuzz-test.yml

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: Runtime Fuzz Testing
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
num_of_iterations:
7+
description: Number of iterations per fuzzable variable
8+
required: False
9+
default: 5
10+
11+
jobs:
12+
build:
13+
runs-on:
14+
- self-hosted
15+
- linux
16+
- x64
17+
- container
18+
19+
container:
20+
image: ghcr.io/intel/fpga-runtime-for-opencl/ubuntu-22.04-dev:main
21+
22+
name: Fuzz Testing
23+
steps:
24+
- name: Set up Python
25+
uses: actions/setup-python@v4
26+
with:
27+
python-version: '3.10'
28+
- name: Clone Radamsa
29+
run: |
30+
mkdir radamsa_repo
31+
cd radamsa_repo
32+
git clone https://gitlab.com/akihe/radamsa.git .
33+
- name: Install Radamsa
34+
run: |
35+
cd radamsa_repo
36+
make
37+
sudo make install
38+
cd ..
39+
- name: Install PyYAML
40+
run: pip install pyyaml
41+
- name: Checkout runtime
42+
uses: actions/checkout@v3
43+
- name: Build
44+
run: |
45+
mkdir -p build/fuzz_testing
46+
cd build/fuzz_testing
47+
CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
48+
ninja -v
49+
- name: Fuzz testing
50+
run: |
51+
cd build/fuzz_testing
52+
ls
53+
cd fuzz_testing/script
54+
export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
55+
NUM_OF_ITERATIONS=${{ github.event.inputs.num_of_iterations }}
56+
# This if block is only used during testing, because if this workflow is triggered via pull_request, ${{ github.event.inputs.num_of_iterations }} would be empty
57+
if [ -z "${NUM_OF_ITERATIONS}" ]; then
58+
NUM_OF_ITERATIONS=1
59+
fi
60+
python3 fuzz_test.py --all -n $NUM_OF_ITERATIONS
61+
- name: Peek results
62+
run: |
63+
cat build/fuzz_testing/fuzz_testing/results/results.yml
64+
- name: Upload results
65+
uses: actions/upload-artifact@v3
66+
with:
67+
name: fpga-runtime-for-opencl-${{ github.sha }}-fuzz-test-results-${{ github.run_id }}
68+
path: |
69+
/__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/results/results.yml
70+
/__w/fpga-runtime-for-opencl/fpga-runtime-for-opencl/build/fuzz_testing/fuzz_testing/test_outputs
71+
if-no-files-found: error

CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,3 +404,10 @@ install(FILES
404404

405405
add_subdirectory(lib)
406406
add_subdirectory(test)
407+
408+
# Flag for building fuzz tests
409+
option(FUZZ_TESTING "Build fuzz tests" OFF)
410+
message(STATUS "Build fuzz tests: ${FUZZ_TESTING}")
411+
if(FUZZ_TESTING)
412+
add_subdirectory(fuzz_testing)
413+
endif()

fuzz_testing/CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (C) 2021 Intel Corporation
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
4+
add_subdirectory(test)
5+
file(COPY original_inputs DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
6+
file(COPY script DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

fuzz_testing/README.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Fuzz Testing
2+
3+
## Context
4+
"In programming and software development, fuzzing or fuzz testing is an automated software testing technique that involves providing invalid, unexpected, or random data as inputs to a computer program. The program is then monitored for exceptions such as crashes, failing built-in code assertions, or potential memory leaks."
5+
6+
## How to do fuzz testing on Github
7+
1. Click on the `Actions` tab
8+
2. Click on `Runtime Fuzz Testing` workflow
9+
3. Click on `Run workflow` (Dropdown button)
10+
4. Enter the number of iterations you want a single variable to be mutated
11+
5. Click on `Run workflow` (Green button)
12+
6. Wait until the test finishes
13+
7. Inside the workflow run, click `Peek results` to look at the results
14+
8. Download the artifact from the `Job Summary` to view the full output & result
15+
16+
## How to do fuzz testing locally
17+
1. Follow [Prerequisites](https://github.com/intel/fpga-runtime-for-opencl#prerequisites) & [Build](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) & [Test](https://github.com/intel/fpga-runtime-for-opencl#building-the-runtime) instructions to get a working build. Make sure that the unit tests finishes successfully.
18+
2. Install Radamsa 0.6+, Python 3.10.7+ and Pyyaml/6.0+
19+
3. Run the following at the top level in the runtime repo
20+
```
21+
mkdir -p build/fuzz_testing
22+
cd build/fuzz_testing
23+
CC="${CC:-gcc}" CXX="${CXX:-g++}" cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug -DACL_CODE_COVERAGE=ON -DACL_TSAN=OFF -DACL_WITH_ASAN=ON -DFUZZ_TESTING=ON "$@"
24+
ninja -v
25+
cd fuzz_testing/script
26+
export AOCL_BOARD_PACKAGE_ROOT="$(git rev-parse --show-toplevel)/test/board/a10_ref"
27+
python3 fuzz_test.py --all
28+
```
29+
4. A results directory and a test_output directory will be created after the fuzz test finishes
30+
31+
## Classifying Failures
32+
- Address Sanitizer Error (ASAN Errors): Any errors caught by address sanitizer such as memory leaks (Not acceptable)
33+
- Aborted Runs: The test aborted/crashed during execution (Not acceptable)
34+
- Assertion Failures: The test failed due to an assertion (Acceptable)
35+
- Failed Runs: The test failed because the CHECK statements failed (Acceptable)
36+
- Hangs: The program did not terminate (Not acceptable)
37+
- Successful runs: The test succeeded (Acceptable)
38+
- Test errors: The test failed due to bugs in the fuzz testing infrastructure (Acceptable)
39+
40+
## Additional Notes
41+
- Currently we are doing most of the fuzz testing on auto discovery strings because these strings are indirectly given by the user.
42+
- The fuzz tests are initialized based on the current unit tests, however they should be regarded as separate tests. (i.e. You can make fuzz_test/unit_test only changes)

fuzz_testing/fuzz_src/fuzz_testing.h

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
/*
2+
This header file contains all the functions that are needed for fuzz testing
3+
*/
4+
5+
#ifndef FUZZ_TESTING_H
6+
#define FUZZ_TESTING_H
7+
#include <fstream>
8+
#include <iostream>
9+
#include <sstream>
10+
#include <string>
11+
#include <unordered_map>
12+
#include <vector>
13+
14+
using namespace std;
15+
16+
#define YML ".yml"
17+
#define TAB " "
18+
#define TAB_LENGTH_FOR_VAR 4
19+
20+
#define FUZZ_TEST_ERROR "Fuzz test error: "
21+
22+
// groupName-testname-varName => data
23+
inline unordered_map<string, string> data_map;
24+
25+
// Basically find terminating '"' that concludes the variable value
26+
inline bool parseVariableValue(ifstream &inputFile, string &returnVal) {
27+
string line;
28+
while (getline(inputFile, line)) {
29+
returnVal = returnVal + "\n" + line;
30+
// Terminating '"' found
31+
if (line[line.length() - 1] == '"') {
32+
// Remove double quotes
33+
returnVal = returnVal.substr(1, returnVal.length() - 2);
34+
return true;
35+
}
36+
}
37+
// EOF
38+
return false;
39+
}
40+
41+
// Generate a string that combines the groupName testName and varName (i.e.
42+
// groupName-testname-varName)
43+
inline string generate_key(const vector<string> &names) {
44+
if (names.size() != 3) {
45+
cout << FUZZ_TEST_ERROR << "Incorrect key size" << endl;
46+
exit(1);
47+
}
48+
return names[0] + "--" + names[1] + "--" + names[2];
49+
}
50+
51+
// A utility function that counts how many "TAB"s are in the current line
52+
// A line should only have at most 2 tabs in the current schema
53+
inline int countTabs(const string &line) {
54+
int count = 0;
55+
for (unsigned int i = 0; i < line.size(); i++) {
56+
if (line[i] == ' ') {
57+
count++;
58+
} else {
59+
if (count % 2 != 0) {
60+
cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
61+
<< endl;
62+
exit(1);
63+
}
64+
int indent = count / 2;
65+
if (indent > 2) {
66+
cout << FUZZ_TEST_ERROR << "Yaml has too much indentation " << endl;
67+
exit(1);
68+
}
69+
return indent;
70+
}
71+
}
72+
// A line of spaces, yaml error
73+
cout << FUZZ_TEST_ERROR << "Yaml should not have an empty line " << endl;
74+
exit(1);
75+
}
76+
77+
// A function that loads fuzzed data from a input yaml file to data_map
78+
inline void preload(ifstream &inputFile) {
79+
string line;
80+
vector<string> names;
81+
// 0 => group, 1 => test, 2 => variable
82+
int lastIndent = 0;
83+
bool first = true;
84+
85+
while (getline(inputFile, line)) {
86+
// Parsing yaml file manually
87+
int currentIndent = countTabs(line);
88+
if (currentIndent - 1 > lastIndent) {
89+
cout << FUZZ_TEST_ERROR << "Yaml does not have correct indentation "
90+
<< endl;
91+
exit(1);
92+
}
93+
int diff = lastIndent - currentIndent;
94+
// Delete leaf if exiting
95+
if (diff >= 0 && !first) {
96+
names.resize(names.size() - diff - 1);
97+
}
98+
stringstream ss_line(line);
99+
string curr_name;
100+
ss_line >> curr_name;
101+
// If current indent level is at group/test
102+
if (currentIndent <= 1) {
103+
// Remove : at the end and add them to names
104+
names.push_back(curr_name.substr(0, curr_name.length() - 1));
105+
} else {
106+
// Else the current indent level must be at variable
107+
stringstream ss(line);
108+
string varName;
109+
string varValue;
110+
// Note: varName contains ':'
111+
ss >> varName;
112+
names.push_back(varName.substr(0, varName.length() - 1));
113+
// Note that varValue contains ""
114+
getline(ss, varValue);
115+
// Bypass leading space
116+
varValue = varValue.substr(1, varValue.size() - 1);
117+
string key = generate_key(names);
118+
if (varValue[varValue.length() - 1] == '"') {
119+
// Filter out double quote characters
120+
data_map.insert(pair<string, string>(
121+
key, varValue.substr(1, varValue.length() - 2)));
122+
} else {
123+
// The variable value contains newline character
124+
// Needs to parse multiple lines until finding the terminating '"'
125+
if (parseVariableValue(inputFile, varValue)) {
126+
data_map.insert(pair<string, string>(key, varValue));
127+
}
128+
// EOF reached but still couldn't find terminating "
129+
else {
130+
cout << FUZZ_TEST_ERROR
131+
<< "EOF reached but still couldn't find terminating \"" << endl;
132+
exit(1);
133+
}
134+
}
135+
}
136+
lastIndent = currentIndent;
137+
first = false;
138+
}
139+
}
140+
141+
// A top level function that preloads all fuzzed data into the data_map
142+
inline void preload_data(string fileName) {
143+
cout << "\nPreloading fuzzed data starts: " << endl;
144+
string pathPrefix = "../mutated_inputs/";
145+
string path = pathPrefix + fileName + YML;
146+
cout << "Opening input file: " << path << endl;
147+
ifstream inputFile(path.c_str());
148+
string value;
149+
if (inputFile.is_open()) {
150+
preload(inputFile);
151+
cout << "Preloading fuzzed data ends " << endl;
152+
} else {
153+
cout << FUZZ_TEST_ERROR << "Unable to open fuzz test file! " << endl;
154+
cout << "Make sure you run the fuzz test in the test directory" << endl;
155+
exit(1);
156+
}
157+
}
158+
159+
// Parameters: group name, test name, variable name
160+
inline string load_fuzzed_value(string groupName, string testName,
161+
string varName) {
162+
vector<string> names = {groupName, testName, varName};
163+
string key = generate_key(names);
164+
if (data_map.find(key) == data_map.end()) {
165+
cout << FUZZ_TEST_ERROR << "Unable to find " << key << endl;
166+
exit(1);
167+
}
168+
return data_map[key];
169+
}
170+
171+
// Cast data to corresponding data types
172+
template <class castType>
173+
inline castType load_fuzzed_value_cast(string groupName, string testName,
174+
string varName) {
175+
try {
176+
// Convert to unsigned long long then convert to castType
177+
string value = load_fuzzed_value(groupName, testName, varName);
178+
int base = 10;
179+
if (value.size() > 2 && value.substr(0, 2) == "0x") {
180+
base = 16;
181+
}
182+
return (castType)stoull(value, 0, base);
183+
} catch (string e) {
184+
cout << FUZZ_TEST_ERROR << "Data type mismatch " << endl;
185+
cout << e << endl;
186+
exit(1);
187+
}
188+
}
189+
190+
// Utility functions
191+
inline bool check_condition(bool condition, bool &check) {
192+
check = condition;
193+
return check;
194+
}
195+
#endif

0 commit comments

Comments
 (0)