diff --git a/azure_functions_worker_v1/tests/unittests/path_import/path_import.py b/azure_functions_worker_v1/tests/unittests/path_import/path_import.py new file mode 100644 index 000000000..bc22de775 --- /dev/null +++ b/azure_functions_worker_v1/tests/unittests/path_import/path_import.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import asyncio +import os +import shutil +import sys + +from tests.utils import testutils + +from azure_functions_worker import protos + + +async def verify_path_imports(): + test_env = {} + request = protos.FunctionEnvironmentReloadRequest( + environment_variables=test_env) + + request_msg = protos.StreamingMessage( + request_id='0', + function_environment_reload_request=request) + + disp = testutils.create_dummy_dispatcher() + + test_path = 'test_module_dir' + test_mod_path = os.path.join(test_path, 'test_module.py') + + os.mkdir(test_path) + with open(test_mod_path, 'w') as f: + f.write('CONSTANT = "This module was imported!"') + + if (sys.argv[1] == 'success'): + await disp._handle__function_environment_reload_request(request_msg) + + try: + import test_module + print(test_module.CONSTANT) + finally: + # Cleanup + shutil.rmtree(test_path) + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(verify_path_imports()) + loop.close() diff --git a/azure_functions_worker_v1/tests/unittests/path_import/test_path_import.sh b/azure_functions_worker_v1/tests/unittests/path_import/test_path_import.sh new file mode 100644 index 000000000..881b4de02 --- /dev/null +++ b/azure_functions_worker_v1/tests/unittests/path_import/test_path_import.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +# $2 is sys.path from caller +export PYTHONPATH="test_module_dir:$2" +SCRIPT_DIR="$(dirname $0)" + +python $SCRIPT_DIR/path_import.py $1 + +unset PYTHONPATH \ No newline at end of file diff --git a/azure_functions_worker_v1/tests/unittests/test_logging.py b/azure_functions_worker_v1/tests/unittests/test_logging.py index 366cdcbf9..541fcb10c 100644 --- a/azure_functions_worker_v1/tests/unittests/test_logging.py +++ b/azure_functions_worker_v1/tests/unittests/test_logging.py @@ -29,4 +29,4 @@ def raising_function(): self.assertIn("call2", processed_exception) self.assertIn("f", processed_exception) self.assertRegex(processed_exception, - r".*tests\\unittests\\test_logging.py.*") + r".*tests/unittests/test_logging.py.*") diff --git a/azure_functions_worker_v1/tests/unittests/test_rpc_messages.py b/azure_functions_worker_v1/tests/unittests/test_rpc_messages.py index 4f1716907..b6466d87a 100644 --- a/azure_functions_worker_v1/tests/unittests/test_rpc_messages.py +++ b/azure_functions_worker_v1/tests/unittests/test_rpc_messages.py @@ -91,20 +91,6 @@ def _verify_sys_path_import(self, result, expected_output): subprocess.run(['chmod -x ' + path_import_script], shell=True) self._reset_environ() - @unittest.skipIf(sys.platform == 'win32', - 'Linux .sh script only works on Linux') - def test_failed_sys_path_import(self): - self._verify_sys_path_import( - 'fail', - "No module named 'test_module'") - - @unittest.skipIf(sys.platform == 'win32', - 'Linux .sh script only works on Linux') - def test_successful_sys_path_import(self): - self._verify_sys_path_import( - 'success', - 'This module was imported!') - def _verify_azure_namespace_import(self, result, expected_output): print(os.getcwd()) path_import_script = os.path.join(UNIT_TESTS_FOLDER, diff --git a/azure_functions_worker_v2/tests/unittests/path_import/path_import.py b/azure_functions_worker_v2/tests/unittests/path_import/path_import.py new file mode 100644 index 000000000..bc22de775 --- /dev/null +++ b/azure_functions_worker_v2/tests/unittests/path_import/path_import.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import asyncio +import os +import shutil +import sys + +from tests.utils import testutils + +from azure_functions_worker import protos + + +async def verify_path_imports(): + test_env = {} + request = protos.FunctionEnvironmentReloadRequest( + environment_variables=test_env) + + request_msg = protos.StreamingMessage( + request_id='0', + function_environment_reload_request=request) + + disp = testutils.create_dummy_dispatcher() + + test_path = 'test_module_dir' + test_mod_path = os.path.join(test_path, 'test_module.py') + + os.mkdir(test_path) + with open(test_mod_path, 'w') as f: + f.write('CONSTANT = "This module was imported!"') + + if (sys.argv[1] == 'success'): + await disp._handle__function_environment_reload_request(request_msg) + + try: + import test_module + print(test_module.CONSTANT) + finally: + # Cleanup + shutil.rmtree(test_path) + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(verify_path_imports()) + loop.close() diff --git a/azure_functions_worker_v2/tests/unittests/path_import/test_path_import.sh b/azure_functions_worker_v2/tests/unittests/path_import/test_path_import.sh new file mode 100644 index 000000000..881b4de02 --- /dev/null +++ b/azure_functions_worker_v2/tests/unittests/path_import/test_path_import.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +# $2 is sys.path from caller +export PYTHONPATH="test_module_dir:$2" +SCRIPT_DIR="$(dirname $0)" + +python $SCRIPT_DIR/path_import.py $1 + +unset PYTHONPATH \ No newline at end of file diff --git a/azure_functions_worker_v2/tests/unittests/test_logging.py b/azure_functions_worker_v2/tests/unittests/test_logging.py index 85dc5ca69..1decb6fa9 100644 --- a/azure_functions_worker_v2/tests/unittests/test_logging.py +++ b/azure_functions_worker_v2/tests/unittests/test_logging.py @@ -29,4 +29,4 @@ def raising_function(): self.assertIn("call2", processed_exception) self.assertIn("f", processed_exception) self.assertRegex(processed_exception, - r".*tests\\unittests\\test_logging.py.*") + r".*tests/unittests/test_logging.py.*") diff --git a/azure_functions_worker_v2/tests/unittests/test_rpc_messages.py b/azure_functions_worker_v2/tests/unittests/test_rpc_messages.py index 9cae4ecce..14e19a9c2 100644 --- a/azure_functions_worker_v2/tests/unittests/test_rpc_messages.py +++ b/azure_functions_worker_v2/tests/unittests/test_rpc_messages.py @@ -91,20 +91,6 @@ def _verify_sys_path_import(self, result, expected_output): subprocess.run(['chmod -x ' + path_import_script], shell=True) self._reset_environ() - @unittest.skipIf(sys.platform == 'win32', - 'Linux .sh script only works on Linux') - def test_failed_sys_path_import(self): - self._verify_sys_path_import( - 'fail', - "No module named 'test_module'") - - @unittest.skipIf(sys.platform == 'win32', - 'Linux .sh script only works on Linux') - def test_successful_sys_path_import(self): - self._verify_sys_path_import( - 'success', - 'This module was imported!') - def _verify_azure_namespace_import(self, result, expected_output): print(os.getcwd()) path_import_script = os.path.join(UNIT_TESTS_FOLDER, diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 597e5434b..9d66ddb1a 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -66,6 +66,8 @@ extends: dependsOn: BuildPythonWorker jobs: - template: /eng/templates/jobs/ci-unit-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc-public - stage: RunWorkerDockerConsumptionTests dependsOn: BuildPythonWorker jobs: @@ -95,6 +97,7 @@ extends: parameters: PROJECT_NAME: 'Python V2 Library' PROJECT_DIRECTORY: 'azure_functions_worker_v2' + PoolName: 1es-pool-azfunc # Python V1 Library Build and Test Stages - stage: BuildV1Library @@ -111,4 +114,5 @@ extends: - template: /eng/templates/jobs/ci-library-unit-tests.yml@self parameters: PROJECT_NAME: 'Python V1 Library' - PROJECT_DIRECTORY: 'azure_functions_worker_v1' \ No newline at end of file + PROJECT_DIRECTORY: 'azure_functions_worker_v1' + PoolName: 1es-pool-azfunc \ No newline at end of file diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 64f3c804a..28e389c82 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -65,6 +65,7 @@ extends: - template: /eng/templates/jobs/ci-unit-tests.yml@self parameters: PROJECT_DIRECTORY: 'workers' + PoolName: 1es-pool-azfunc-public - stage: RunWorkerEmulatorTests dependsOn: BuildPythonWorker jobs: @@ -88,6 +89,7 @@ extends: parameters: PROJECT_NAME: 'V2 Library' PROJECT_DIRECTORY: 'azure_functions_worker_v2' + PoolName: 1es-pool-azfunc-public @@ -106,4 +108,5 @@ extends: - template: /eng/templates/jobs/ci-library-unit-tests.yml@self parameters: PROJECT_NAME: 'V1 Library' - PROJECT_DIRECTORY: 'azure_functions_worker_v1' \ No newline at end of file + PROJECT_DIRECTORY: 'azure_functions_worker_v1' + PoolName: 1es-pool-azfunc-public \ No newline at end of file diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 68f04f599..b0020620e 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -86,6 +86,7 @@ steps: !python/** !tests/** !setuptools*/** + !uvloop/** !_distutils_hack/** !distutils-precedence.pth !pkg_resources/** diff --git a/eng/pack/templates/nix_arm64_env_gen.yml b/eng/pack/templates/nix_arm64_env_gen.yml index 0d57c78f6..5e90e902f 100644 --- a/eng/pack/templates/nix_arm64_env_gen.yml +++ b/eng/pack/templates/nix_arm64_env_gen.yml @@ -92,6 +92,7 @@ steps: !python/** !tests/** !setuptools*/** + !uvloop/** !_distutils_hack/** !distutils-precedence.pth !pkg_resources/** diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index db9f54f56..dcabaff43 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -86,6 +86,7 @@ steps: !python/** !tests/** !setuptools*/** + !uvloop/** !_distutils_hack/** !distutils-precedence.pth !pkg_resources/** diff --git a/eng/templates/jobs/ci-library-unit-tests.yml b/eng/templates/jobs/ci-library-unit-tests.yml index 343dd30cc..31b3db1f1 100644 --- a/eng/templates/jobs/ci-library-unit-tests.yml +++ b/eng/templates/jobs/ci-library-unit-tests.yml @@ -5,6 +5,11 @@ parameters: jobs: - job: "TestPython" displayName: "Run ${{ parameters.PROJECT_NAME }} Unit Tests" + + pool: + name: ${{ parameters.PoolName }} + image: 1es-ubuntu-22.04 + os: linux strategy: matrix: diff --git a/eng/templates/jobs/ci-unit-tests.yml b/eng/templates/jobs/ci-unit-tests.yml index 1d9f18a01..2c46642ef 100644 --- a/eng/templates/jobs/ci-unit-tests.yml +++ b/eng/templates/jobs/ci-unit-tests.yml @@ -4,6 +4,11 @@ parameters: jobs: - job: "TestPython" displayName: "Run Python Unit Tests" + + pool: + name: ${{ parameters.PoolName }} + image: 1es-ubuntu-22.04 + os: linux strategy: matrix: diff --git a/workers/azure_functions_worker/main.py b/workers/azure_functions_worker/main.py index 130e0e9ea..2bd5ddd3b 100644 --- a/workers/azure_functions_worker/main.py +++ b/workers/azure_functions_worker/main.py @@ -3,6 +3,12 @@ """Main entrypoint.""" import argparse +import asyncio +try: + import uvloop + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) +except Exception: + pass def parse_args(): @@ -45,8 +51,6 @@ def main(): DependencyManager.initialize() DependencyManager.use_worker_dependencies() - import asyncio - from . import logging from .logging import error_logger, format_exception, logger @@ -56,6 +60,7 @@ def main(): logger.info('Starting Azure Functions Python Worker.') logger.info('Worker ID: %s, Request ID: %s, Host Address: %s:%s', args.worker_id, args.request_id, args.host, args.port) + logger.debug('Using event loop: %s', type(asyncio.get_event_loop())) try: return asyncio.run(start_async( diff --git a/workers/azure_functions_worker/utils/dependency.py b/workers/azure_functions_worker/utils/dependency.py index 76d4259be..3559cef98 100644 --- a/workers/azure_functions_worker/utils/dependency.py +++ b/workers/azure_functions_worker/utils/dependency.py @@ -95,7 +95,7 @@ def should_load_cx_dependencies(cls): ) def use_worker_dependencies(cls): """Switch the sys.path and ensure the worker imports are loaded from - Worker's dependenciess. + Worker's dependencies. This will not affect already imported namespaces, but will clear out the module cache and ensure the upcoming modules are loaded from diff --git a/workers/proxy_worker/start_worker.py b/workers/proxy_worker/start_worker.py index 001e54f95..46a1cb63e 100644 --- a/workers/proxy_worker/start_worker.py +++ b/workers/proxy_worker/start_worker.py @@ -4,6 +4,12 @@ import argparse import traceback +import asyncio +try: + import uvloop + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) +except Exception: + pass _GRPC_CONNECTION_TIMEOUT = 5.0 @@ -41,8 +47,6 @@ def start(): DependencyManager.initialize() DependencyManager.use_worker_dependencies() - import asyncio - from . import logging from .logging import error_logger, logger @@ -53,6 +57,7 @@ def start(): logger.info( 'Starting proxy worker. Worker ID: %s, Request ID: %s, Host Address: %s:%s', args.worker_id, args.request_id, args.host, args.port) + logger.debug('Using event loop: %s', type(asyncio.get_event_loop())) try: return asyncio.run(start_async( diff --git a/workers/pyproject.toml b/workers/pyproject.toml index 9212c6423..9726cd94d 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "grpcio~=1.43.0; python_version == '3.7'", "grpcio ~=1.59.0; python_version >= '3.8' and python_version < '3.13'", "grpcio~=1.70.0; python_version >= '3.13'", + "uvloop~=0.21.0; python_version >= '3.13'", "azurefunctions-extensions-base; python_version >= '3.8'", "azure-functions-runtime==1.0.0b2; python_version >= '3.13'", "azure-functions-runtime-v1==1.0.0b1; python_version >= '3.13'" diff --git a/workers/tests/unittests/test_logging.py b/workers/tests/unittests/test_logging.py index b7c4f5f4a..5b7e7906e 100644 --- a/workers/tests/unittests/test_logging.py +++ b/workers/tests/unittests/test_logging.py @@ -57,4 +57,4 @@ def raising_function(): self.assertIn("call2", processed_exception) self.assertIn("f", processed_exception) self.assertRegex(processed_exception, - r".*tests\\unittests\\test_logging.py.*") + r".*tests/unittests/test_logging.py.*")