Skip to content

Commit 70b2475

Browse files
pdthummargavin-aguiar
pdthummar
and
gavin-aguiar
authored
Fixing E2E and UT flaky tests. (#1188)
* Fixing E2E and UT flaky tests. * addressing comments * running tests parallel and increasing the verbosity of output --------- Co-authored-by: gavin-aguiar <[email protected]>
1 parent 5b3a930 commit 70b2475

16 files changed

+8
-75
lines changed

‎.github/workflows/ci_e2e_workflow.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ jobs:
5858
}
5959
6060
python -m pip install --upgrade pip
61-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
61+
python -m pip install -U -e .[dev]
6262
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
6363
6464
# Retry a couple times to avoid certificate issue
@@ -76,7 +76,7 @@ jobs:
7676
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
7777
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
7878
run: |
79-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
79+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
8080
- name: Running 3.8 Tests
8181
if: matrix.python-version == 3.8
8282
env:
@@ -88,7 +88,7 @@ jobs:
8888
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
8989
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
9090
run: |
91-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
91+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
9292
- name: Running 3.9 Tests
9393
if: matrix.python-version == 3.9
9494
env:
@@ -100,7 +100,7 @@ jobs:
100100
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
101101
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
102102
run: |
103-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
103+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
104104
- name: Running 3.10 Tests
105105
if: matrix.python-version == 3.10
106106
env:
@@ -112,7 +112,7 @@ jobs:
112112
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
113113
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
114114
run: |
115-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
115+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
116116
- name: Running 3.11 Tests
117117
if: matrix.python-version == 3.11
118118
env:
@@ -124,7 +124,7 @@ jobs:
124124
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
125125
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
126126
run: |
127-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
127+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
128128
- name: Codecov
129129
uses: codecov/[email protected]
130130
with:

‎.github/workflows/ci_ut_workflow.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
}
5656
5757
python -m pip install --upgrade pip
58-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
58+
python -m pip install -U -e .[dev]
5959
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
6060
6161
# Retry a couple times to avoid certificate issue
@@ -66,7 +66,7 @@ jobs:
6666
env:
6767
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} # needed for installing azure-functions-durable while running setup.py
6868
run: |
69-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
69+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
7070
- name: Codecov
7171
uses: codecov/codecov-action@v3
7272
with:

‎tests/endtoend/test_blob_functions.py

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ class TestBlobFunctions(testutils.WebHostTestCase):
1111
def get_script_dir(cls):
1212
return testutils.E2E_TESTS_FOLDER / 'blob_functions'
1313

14-
@testutils.retryable_test(3, 5)
1514
def test_blob_io_str(self):
1615
r = self.webhost.request('POST', 'put_blob_str', data='test-data')
1716
self.assertEqual(r.status_code, 200)
@@ -25,7 +24,6 @@ def test_blob_io_str(self):
2524
self.assertEqual(r.status_code, 200)
2625
self.assertEqual(r.text, 'test-data')
2726

28-
@testutils.retryable_test(3, 5)
2927
def test_blob_io_large_str(self):
3028
large_string = 'DummyDataDummyDataDummyData' * 1024 * 1024 # 27 MB
3129

@@ -41,7 +39,6 @@ def test_blob_io_large_str(self):
4139
self.assertEqual(r.status_code, 200)
4240
self.assertEqual(r.text, large_string)
4341

44-
@testutils.retryable_test(3, 5)
4542
def test_blob_io_bytes(self):
4643
r = self.webhost.request('POST', 'put_blob_bytes',
4744
data='test-dată'.encode('utf-8'))
@@ -56,7 +53,6 @@ def test_blob_io_bytes(self):
5653
self.assertEqual(r.status_code, 200)
5754
self.assertEqual(r.text, 'test-dată')
5855

59-
@testutils.retryable_test(3, 5)
6056
def test_blob_io_large_bytes(self):
6157
large_string = 'DummyDataDummyDataDummyData' * 1024 * 1024 # 27 MB
6258

@@ -73,7 +69,6 @@ def test_blob_io_large_bytes(self):
7369
self.assertEqual(r.status_code, 200)
7470
self.assertEqual(r.text, large_string)
7571

76-
@testutils.retryable_test(3, 5)
7772
def test_blob_io_filelike(self):
7873
r = self.webhost.request('POST', 'put_blob_filelike')
7974
self.assertEqual(r.status_code, 200)
@@ -83,7 +78,6 @@ def test_blob_io_filelike(self):
8378
self.assertEqual(r.status_code, 200)
8479
self.assertEqual(r.text, 'filelike')
8580

86-
@testutils.retryable_test(3, 5)
8781
def test_blob_io_return(self):
8882
r = self.webhost.request('POST', 'put_blob_return')
8983
self.assertEqual(r.status_code, 200)
@@ -92,7 +86,6 @@ def test_blob_io_return(self):
9286
self.assertEqual(r.status_code, 200)
9387
self.assertEqual(r.text, 'FROM RETURN')
9488

95-
@testutils.retryable_test(3, 5)
9689
def test_blob_trigger(self):
9790
data = "DummyData"
9891

@@ -126,7 +119,6 @@ def test_blob_trigger(self):
126119
if try_no == max_retries - 1:
127120
raise
128121

129-
@testutils.retryable_test(3, 5)
130122
def test_blob_trigger_with_large_content(self):
131123
data = 'DummyDataDummyDataDummyData' * 1024 * 1024 # 27 MB
132124

‎tests/endtoend/test_blueprint_functions.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ def get_script_dir(cls):
1010
return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
1111
'functions_in_blueprint_only'
1212

13-
@testutils.retryable_test(3, 5)
1413
def test_function_in_blueprint_only(self):
1514
r = self.webhost.request('GET', 'default_template')
1615
self.assertTrue(r.ok)
@@ -22,7 +21,6 @@ def get_script_dir(cls):
2221
return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
2322
'functions_in_both_blueprint_functionapp'
2423

25-
@testutils.retryable_test(3, 5)
2624
def test_functions_in_both_blueprint_functionapp(self):
2725
r = self.webhost.request('GET', 'default_template')
2826
self.assertTrue(r.ok)
@@ -37,7 +35,6 @@ def get_script_dir(cls):
3735
return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
3836
'multiple_function_registers'
3937

40-
@testutils.retryable_test(3, 5)
4138
def test_function_in_blueprint_only(self):
4239
r = self.webhost.request('GET', 'return_http')
4340
self.assertEqual(r.status_code, 404)
@@ -49,7 +46,6 @@ def get_script_dir(cls):
4946
return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' / \
5047
'only_blueprint'
5148

52-
@testutils.retryable_test(3, 5)
5349
def test_only_blueprint(self):
5450
"""Test if the default template of Http trigger in Python
5551
Function app

‎tests/endtoend/test_cosmosdb_functions.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ def test_cosmosdb_trigger(self):
4949
else:
5050
break
5151

52-
@testutils.retryable_test(3, 5)
5352
def test_cosmosdb_input(self):
5453
time.sleep(5)
5554
data = str(round(time.time()))

‎tests/endtoend/test_dependency_isolation_functions.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,13 @@ def tearDownClass(self):
5252
def get_script_dir(cls):
5353
return cls.project_root
5454

55-
@testutils.retryable_test(3, 5)
5655
def test_dependency_function_should_return_ok(self):
5756
"""The common scenario of general import should return OK in any
5857
circumstances
5958
"""
6059
r: Response = self.webhost.request('GET', 'report_dependencies')
6160
self.assertTrue(r.ok)
6261

63-
@testutils.retryable_test(3, 5)
6462
def test_feature_flag_is_turned_on(self):
6563
"""Since passing the feature flag PYTHON_ISOLATE_WORKER_DEPENDENCIES to
6664
the host, the customer's function should also be able to receive it
@@ -70,7 +68,6 @@ def test_feature_flag_is_turned_on(self):
7068
flag_value = environments['PYTHON_ISOLATE_WORKER_DEPENDENCIES']
7169
self.assertEqual(flag_value, '1')
7270

73-
@testutils.retryable_test(3, 5)
7471
def test_working_directory_resolution(self):
7572
"""Check from the dependency manager and see if the current working
7673
directory is resolved correctly
@@ -88,7 +85,6 @@ def test_working_directory_resolution(self):
8885
is_envvar_true(PYAZURE_INTEGRATION_TEST),
8986
'Integration test expects dependencies derived from core tools folder'
9087
)
91-
@testutils.retryable_test(3, 5)
9288
def test_paths_resolution(self):
9389
"""Dependency manager requires paths to be resolved correctly before
9490
switching to customer's modules. This test is to ensure when the app
@@ -112,7 +108,6 @@ def test_paths_resolution(self):
112108
).lower()
113109
)
114110

115-
@testutils.retryable_test(3, 5)
116111
def test_loading_libraries_from_customers_package(self):
117112
"""Since the Python now loaded the customer's dependencies, the
118113
libraries version should match the ones in
@@ -170,7 +165,6 @@ def tearDownClass(self):
170165
def get_script_dir(cls):
171166
return cls.project_root
172167

173-
@testutils.retryable_test(3, 5)
174168
def test_loading_libraries_from_customers_package(self):
175169
r: Response = self.webhost.request('GET', 'report_dependencies')
176170
libraries = r.json()['libraries']
@@ -219,7 +213,6 @@ def tearDownClass(self):
219213
def get_script_dir(cls):
220214
return cls.project_root
221215

222-
@testutils.retryable_test(3, 5)
223216
def test_loading_libraries_from_customers_package(self):
224217
r: Response = self.webhost.request('GET', 'report_dependencies')
225218
libraries = r.json()['libraries']

‎tests/endtoend/test_eventgrid_functions.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ def eventgrid_webhook_request(self, meth, funcname, *args, **kwargs):
2727
return request_method(url, *args, params=params, headers=headers,
2828
**kwargs)
2929

30-
@testutils.retryable_test(3, 5)
3130
@unittest.skip("Run locally. Running on Azure fails with 401/403 as the"
3231
"host does not pick up the SecretKey from the"
3332
"azure_functions_worker.testutils.py.SECRETS_TEMPLATE and"
@@ -92,7 +91,6 @@ def test_eventgrid_trigger(self):
9291
else:
9392
break
9493

95-
@testutils.retryable_test(3, 5)
9694
def test_eventgrid_output_binding(self):
9795
"""test event_grid output binding
9896

‎tests/endtoend/test_eventhub_batch_functions.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ class TestEventHubFunctions(testutils.WebHostTestCase):
2222
def get_script_dir(cls):
2323
return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions'
2424

25-
@testutils.retryable_test(3, 5)
2625
def test_eventhub_multiple(self):
2726
NUM_EVENTS = 3
2827
all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
@@ -65,7 +64,6 @@ def test_eventhub_multiple(self):
6564
finally:
6665
self._cleanup(old_partition_key)
6766

68-
@testutils.retryable_test(3, 5)
6967
def test_eventhub_multiple_with_metadata(self):
7068
# Generate a unique event body for EventHub event
7169
# Record the start_time and end_time for checking event enqueue time
@@ -165,7 +163,6 @@ def get_script_dir(cls):
165163
return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' / \
166164
'eventhub_batch_functions_stein'
167165

168-
@testutils.retryable_test(3, 5)
169166
def test_eventhub_multiple(self):
170167
NUM_EVENTS = 3
171168
all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
@@ -199,7 +196,6 @@ def test_eventhub_multiple(self):
199196

200197
self.assertDictEqual(all_row_keys_seen, row_keys_seen)
201198

202-
@testutils.retryable_test(3, 5)
203199
def test_eventhub_multiple_with_metadata(self):
204200
# Generate a unique event body for EventHub event
205201
# Record the start_time and end_time for checking event enqueue time

‎tests/endtoend/test_eventhub_functions.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ class TestEventHubFunctions(testutils.WebHostTestCase):
2222
def get_script_dir(cls):
2323
return testutils.E2E_TESTS_FOLDER / 'eventhub_functions'
2424

25-
@testutils.retryable_test(3, 5)
2625
def test_eventhub_trigger(self):
2726
# Generate a unique event body for the EventHub event
2827
data = str(round(time.time()))
@@ -47,7 +46,6 @@ def test_eventhub_trigger(self):
4746
# Check if the event body matches the initial data
4847
self.assertEqual(response, doc)
4948

50-
@testutils.retryable_test(3, 5)
5149
def test_eventhub_trigger_with_metadata(self):
5250
# Generate a unique event body for EventHub event
5351
# Record the start_time and end_time for checking event enqueue time

‎tests/endtoend/test_http_functions.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,6 @@ def get_script_dir(cls):
139139
return testutils.E2E_TESTS_FOLDER / 'http_functions' / \
140140
'common_libs_functions'
141141

142-
@testutils.retryable_test(3, 5)
143142
def test_numpy(self):
144143
r = self.webhost.request('GET', 'numpy_func',
145144
timeout=REQUEST_TIMEOUT_SEC)
@@ -148,46 +147,40 @@ def test_numpy(self):
148147

149148
self.assertEqual(r.content.decode("UTF-8"), res)
150149

151-
@testutils.retryable_test(3, 5)
152150
def test_requests(self):
153151
r = self.webhost.request('GET', 'requests_func',
154152
timeout=10)
155153

156154
self.assertTrue(r.ok)
157155
self.assertEqual(r.content.decode("UTF-8"), 'req status code: 200')
158156

159-
@testutils.retryable_test(3, 5)
160157
def test_pandas(self):
161158
r = self.webhost.request('GET', 'pandas_func',
162159
timeout=REQUEST_TIMEOUT_SEC)
163160

164161
self.assertIn("two-dimensional",
165162
r.content.decode("UTF-8"))
166163

167-
@testutils.retryable_test(3, 5)
168164
def test_sklearn(self):
169165
r = self.webhost.request('GET', 'sklearn_func',
170166
timeout=REQUEST_TIMEOUT_SEC)
171167

172168
self.assertIn("First 5 records of array:",
173169
r.content.decode("UTF-8"))
174170

175-
@testutils.retryable_test(3, 5)
176171
def test_opencv(self):
177172
r = self.webhost.request('GET', 'opencv_func',
178173
timeout=REQUEST_TIMEOUT_SEC)
179174

180175
self.assertIn("opencv version:",
181176
r.content.decode("UTF-8"))
182177

183-
@testutils.retryable_test(3, 5)
184178
def test_dotenv(self):
185179
r = self.webhost.request('GET', 'dotenv_func',
186180
timeout=REQUEST_TIMEOUT_SEC)
187181

188182
self.assertEqual(r.content.decode("UTF-8"), "found")
189183

190-
@testutils.retryable_test(3, 5)
191184
def test_plotly(self):
192185
r = self.webhost.request('GET', 'plotly_func',
193186
timeout=REQUEST_TIMEOUT_SEC)

0 commit comments

Comments
 (0)