From 012a8a8c75c2b40a7b4869a0d463e68bce8add77 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 16 Aug 2023 17:09:13 -0700 Subject: [PATCH 1/6] Fixes #164 Co-authored-by: Javier Evans --- common/etc/nginx/include/s3gateway.js | 28 +++++++++++-------- .../etc/nginx/templates/default.conf.template | 2 +- test.sh | 5 ++++ test/data/bucket-1/test/index.html | 1 + test/integration/test_api.sh | 13 ++++++++- 5 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 test/data/bucket-1/test/index.html diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index dda556cc..4db6dde5 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -322,9 +322,9 @@ function redirectToS3(r) { if (isDirectoryListing && (r.method === 'GET' || r.method === 'HEAD')) { r.internalRedirect("@s3PreListing"); - } else if ( PROVIDE_INDEX_PAGE == true ) { + } else if (PROVIDE_INDEX_PAGE === true) { r.internalRedirect("@s3"); - } else if ( !ALLOW_LISTING && !PROVIDE_INDEX_PAGE && uriPath == "/" ) { + } else if (!ALLOW_LISTING && !PROVIDE_INDEX_PAGE && uriPath === "/") { r.internalRedirect("@error404"); } else { r.internalRedirect("@s3"); @@ -333,8 +333,12 @@ function redirectToS3(r) { function trailslashControl(r) { if (APPEND_SLASH) { + // For the purposes of understanding whether this is a directory, + // consider the uri without query params or anchors + const path = r.variables.uri_path.split(/[?#]/)[0]; + const hasExtension = /\/[^.\/]+\.[^.]+$/; - if (!hasExtension.test(r.variables.uri_path) && !_isDirectory(r.variables.uri_path)){ + if (!hasExtension.test(path) && !_isDirectory(path)){ return r.internalRedirect("@trailslash"); } } @@ -353,22 +357,20 @@ async function loadContent(r) { r.internalRedirect("@s3Directory"); return; } - const url = s3uri(r); + const uri = s3uri(r); let reply = await ngx.fetch( - `http://127.0.0.1:80${url}` + `http://127.0.0.1:80${uri}` ); - if (reply.status == 200) { - // found index.html, so redirect to it - r.internalRedirect(r.variables.request_uri + INDEX_PAGE); - } else if (reply.status == 404) { - // else just list the contents of the directory + if (reply.status === 200) { + utils.debug_log(r, `Found index file, redirecting to: ${uri}`); + r.internalRedirect(uri); + } else if (reply.status === 404) { + // As there was no index file found, just list the contents of the directory r.internalRedirect("@s3Directory"); } else { r.internalRedirect("@error500"); } - - return; } /** @@ -449,6 +451,8 @@ function _escapeURIPath(uri) { * @private */ function _isDirectory(path) { + // if (!path) return false; + // str.slice(-1); if (path === undefined) { return false; } diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index c78ffc46..7873a190 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -326,7 +326,7 @@ server { location @trailslash { # 302 to request without slashes - rewrite ^ $scheme://$http_host$request_uri/ redirect; + rewrite ^ $scheme://$http_host$uri/$is_args$query_string redirect; } # Provide a hint to the client on 405 errors of the acceptable request methods diff --git a/test.sh b/test.sh index d90f9ce7..20920005 100755 --- a/test.sh +++ b/test.sh @@ -411,6 +411,11 @@ integration_test 2 1 0 0 compose stop nginx-s3-gateway # Restart with new config +p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index" +integration_test 2 1 1 1 + +compose stop nginx-s3-gateway # Restart with new config + p "Testing API with AWS Signature V2 and static site on" integration_test 2 0 1 0 diff --git a/test/data/bucket-1/test/index.html b/test/data/bucket-1/test/index.html new file mode 100644 index 00000000..da85dd89 --- /dev/null +++ b/test/data/bucket-1/test/index.html @@ -0,0 +1 @@ +

This is an index page of the d directory

\ No newline at end of file diff --git a/test/integration/test_api.sh b/test/integration/test_api.sh index ec49f99f..cb17a61c 100644 --- a/test/integration/test_api.sh +++ b/test/integration/test_api.sh @@ -288,6 +288,15 @@ assertHttpRequestEquals "GET" "/statichost/noindexdir/multipledir/" "data/bucket assertHttpRequestEquals "GET" "/statichost" "data/bucket-1/statichost/index.html" assertHttpRequestEquals "GET" "/statichost/noindexdir/multipledir" "data/bucket-1/statichost/noindexdir/multipledir/index.html" fi + + if [ "${allow_directory_list}" == "1" ]; then + if [ "$append_slash" == "1" ]; then + assertHttpRequestEquals "GET" "test" "200" + assertHttpRequestEquals "GET" "test/" "200" + assertHttpRequestEquals "GET" "test?foo=bar" "200" + assertHttpRequestEquals "GET" "test/?foo=bar" "200" + fi + fi fi if [ "${allow_directory_list}" == "1" ]; then @@ -299,7 +308,9 @@ if [ "${allow_directory_list}" == "1" ]; then assertHttpRequestEquals "GET" "%D1%81%D0%B8%D1%81%D1%82%D0%B5%D0%BC%D1%8B/" "200" assertHttpRequestEquals "GET" "системы/" "200" if [ "$append_slash" == "1" ]; then - assertHttpRequestEquals "GET" "b" "302" + if [ "${index_page}" == "0" ]; then + assertHttpRequestEquals "GET" "b" "302" + fi else assertHttpRequestEquals "GET" "b" "404" fi From ecfcb24745e67d09b0b037099e90b6a9e36f5f42 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Fri, 18 Aug 2023 14:49:10 -0700 Subject: [PATCH 2/6] make the isDirectory check simpler and more inclusive of error states --- common/etc/nginx/include/s3gateway.js | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 4db6dde5..2ca140ec 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -451,18 +451,9 @@ function _escapeURIPath(uri) { * @private */ function _isDirectory(path) { - // if (!path) return false; - // str.slice(-1); - if (path === undefined) { - return false; - } - const len = path.length; - - if (len < 1) { - return false; - } + if (!path) return false; - return path.charAt(len - 1) === '/'; + return path.slice(-1) === '/'; } /** From c1fe531512446c27b2ba5eb0756e1a456632b7ba Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 23 Aug 2023 14:54:11 -0700 Subject: [PATCH 3/6] Allow useful output from curl and enable timeouts This change adds three new flags when using curl to hit endpoints as part of integration tests: --connect-timeout 3 --max-time 30 --no-progress-meter --- test/integration/test_api.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/integration/test_api.sh b/test/integration/test_api.sh index cb17a61c..5153233c 100644 --- a/test/integration/test_api.sh +++ b/test/integration/test_api.sh @@ -63,6 +63,7 @@ if ! [ -x "${curl_cmd}" ]; then e "required dependency not found: curl not found in the path or not executable" exit ${no_dep_exit_code} fi +curl_cmd="${curl_cmd} --connect-timeout 3 --max-time 30 --no-progress-meter" # Allow for MacOS which does not support "md5sum" # but has "md5 -r" which can be substituted @@ -104,11 +105,11 @@ assertHttpRequestEquals() { if [ "${method}" = "HEAD" ]; then expected_response_code="$3" - actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' --head "${uri}" ${extra_arg})" + actual_response_code="$(${curl_cmd} -o /dev/null -w '%{http_code}' --head "${uri}" ${extra_arg})" if [ "${expected_response_code}" != "${actual_response_code}" ]; then e "Response code didn't match expectation. Request [${method} ${uri}] Expected [${expected_response_code}] Actual [${actual_response_code}]" - e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' --head '${uri}' ${extra_arg}" + e "curl command: ${curl_cmd} -o /dev/null -w '%{http_code}' --head '${uri}' ${extra_arg}" exit ${test_fail_exit_code} fi elif [ "${method}" = "GET" ]; then @@ -118,21 +119,21 @@ assertHttpRequestEquals() { checksum_output="$(${checksum_cmd} "${body_data_path}")" expected_checksum="${checksum_output:0:${checksum_length}}" - curl_checksum_output="$(${curl_cmd} -s -X "${method}" "${uri}" ${extra_arg} | ${checksum_cmd})" + curl_checksum_output="$(${curl_cmd} -X "${method}" "${uri}" ${extra_arg} | ${checksum_cmd})" s3_file_checksum="${curl_checksum_output:0:${checksum_length}}" if [ "${expected_checksum}" != "${s3_file_checksum}" ]; then e "Checksum doesn't match expectation. Request [${method} ${uri}] Expected [${expected_checksum}] Actual [${s3_file_checksum}]" - e "curl command: ${curl_cmd} -s -X '${method}' '${uri}' ${extra_arg} | ${checksum_cmd}" + e "curl command: ${curl_cmd} -X '${method}' '${uri}' ${extra_arg} | ${checksum_cmd}" exit ${test_fail_exit_code} fi else expected_response_code="$3" - actual_response_code="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' "${uri}" ${extra_arg})" + actual_response_code="$(${curl_cmd} -o /dev/null -w '%{http_code}' "${uri}" ${extra_arg})" if [ "${expected_response_code}" != "${actual_response_code}" ]; then e "Response code didn't match expectation. Request [${method} ${uri}] Expected [${expected_response_code}] Actual [${actual_response_code}]" - e "curl command: ${curl_cmd} -s -o /dev/null -w '%{http_code}' '${uri}' ${extra_arg}" + e "curl command: ${curl_cmd} -o /dev/null -w '%{http_code}' '${uri}' ${extra_arg}" exit ${test_fail_exit_code} fi fi From 13b79e9511b90e094c24146c1477b783f8bb3353 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 6 Sep 2023 10:30:42 -0700 Subject: [PATCH 4/6] Add nginx debug build container This change allows for the test script to be run against a container that is using a custom nginx build compiled with the --with-debug flag. --- Dockerfile.debug | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ test.sh | 28 ++++++++++++------- 2 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 Dockerfile.debug diff --git a/Dockerfile.debug b/Dockerfile.debug new file mode 100644 index 00000000..20f42e49 --- /dev/null +++ b/Dockerfile.debug @@ -0,0 +1,70 @@ +# This container image removes the existing njs package from the inherited image +# (which could be OSS NGINX or NGINX Plus), builds njs from the latest +# source, and installs it. +FROM nginx-s3-gateway + +RUN set -eux \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update -qq; \ + apt-get install --no-install-recommends --no-install-suggests --yes \ + make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev \ + libpcre2-posix3 libxml2-dev libxslt1-dev zlib1g-dev; \ + mkdir -p /tmp/nginx /tmp/njs-latest; \ + curl --retry 6 --location "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ + | gunzip | tar --extract --strip-components=1 --directory /tmp/nginx; \ + curl --retry 6 --location 'https://hg.nginx.org/njs/archive/tip.tar.gz' \ + | gunzip | tar --extract --strip-components=1 --directory /tmp/njs-latest; \ + cd /tmp/njs-latest; \ + ./configure; \ + make -j "$(nproc)"; \ + cp build/njs /usr/bin/njs; \ + cd /tmp/nginx; \ + ./configure \ + --add-dynamic-module=/tmp/njs-latest/nginx \ + --prefix=/etc/nginx \ + --sbin-path=/usr/sbin/nginx \ + --modules-path=/usr/lib/nginx/modules \ + --conf-path=/etc/nginx/nginx.conf \ + --error-log-path=/var/log/nginx/error.log \ + --http-log-path=/var/log/nginx/access.log \ + --pid-path=/var/run/nginx.pid \ + --lock-path=/var/run/nginx.lock \ + --http-client-body-temp-path=/var/cache/nginx/client_temp \ + --http-proxy-temp-path=/var/cache/nginx/proxy_temp \ + --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \ + --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \ + --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ + --user=nginx --group=nginx --with-compat --with-file-aio \ + --with-debug \ + --with-stream \ + --with-threads \ + --with-compat \ + --with-file-aio \ + --with-threads \ + --with-http_addition_module \ + --with-http_auth_request_module \ + --with-http_dav_module \ + --with-http_flv_module \ + --with-http_gunzip_module \ + --with-http_gzip_static_module \ + --with-http_mp4_module \ + --with-http_random_index_module \ + --with-http_realip_module \ + --with-http_secure_link_module \ + --with-http_slice_module \ + --with-http_ssl_module \ + --with-http_stub_status_module \ + --with-http_sub_module \ + --with-http_v2_module \ + --with-stream_realip_module \ + --with-stream_ssl_module \ + --with-stream_ssl_preread_module \ + --with-cc-opt="-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-${NGINX_VERSION}/debian/debuild-base/nginx-${NGINX_VERSION}=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC"; \ + make -j "$(nproc)"; \ + cp objs/ngx_stream_js_module.so /usr/lib/nginx/modules; \ + cp objs/ngx_http_js_module.so /usr/lib/nginx/modules; \ + make install ; \ + apt-get purge --yes --auto-remove make gcc libc6-dev expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev zlib1g-dev; \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* diff --git a/test.sh b/test.sh index 20920005..14f9674f 100755 --- a/test.sh +++ b/test.sh @@ -66,21 +66,25 @@ e() { >&2 echo "$1" } -usage() { e "Usage: $0 [--latest-njs ] [--unprivileged ] [--type " 1>&2; exit 1; } +usage() { e "Usage: $0 [--latest-njs ] [--latest-njs-debug ] [--unprivileged ] [--type ]" 1>&2; exit 1; } for arg in "$@"; do shift case "$arg" in - '--help') set -- "$@" '-h' ;; - '--latest-njs') set -- "$@" '-j' ;; - '--unprivileged') set -- "$@" '-u' ;; - '--type') set -- "$@" '-t' ;; - *) set -- "$@" "$arg" ;; + '--help') set -- "$@" '-h' ;; + '--latest-njs') set -- "$@" '-j' ;; + '--latest-njs-debug') set -- "$@" '-d' ;; + '--unprivileged') set -- "$@" '-u' ;; + '--type') set -- "$@" '-t' ;; + *) set -- "$@" "$arg" ;; esac done -while getopts "hjut:" arg; do +while getopts "hdjut:" arg; do case "${arg}" in + d) + njs_latest_debug="1" + ;; j) njs_latest="1" ;; @@ -109,10 +113,10 @@ else startup_message="Starting NGINX ${nginx_type}" fi -if [ -z "${njs_latest}" ]; then +if [ -z "${njs_latest}" ] && [ -z "${njs_latest_debug}" ]; then njs_latest="0" startup_message="${startup_message} with the release NJS module (default)" -elif [ ${njs_latest} -eq 1 ]; then +elif [[ ${njs_latest} -eq 1 ]] || [[ ${njs_latest_debug} -eq 1 ]]; then startup_message="${startup_message} with the latest NJS module" else startup_message="${startup_message} with the release NJS module" @@ -311,7 +315,11 @@ else --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} . fi -if [ ${njs_latest} -eq 1 ]; then +if [ ${njs_latest_debug} -eq 1 ]; then + p "Layering in latest NJS build and nginx with debug logging" + docker build -f Dockerfile.debug \ + --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} . +elif [ ${njs_latest} -eq 1 ]; then p "Layering in latest NJS build" docker build -f Dockerfile.latest-njs \ --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} . From eeda5ed267fbd46d2971279031dab2057b683b9c Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 6 Sep 2023 10:33:49 -0700 Subject: [PATCH 5/6] Fix latest-njs build in container The newest changes to njs require additional dependencies for compilation. This change adds those dependencies and brings the compiler flags closer to the configuration provided by the OSS packages. --- Dockerfile.latest-njs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Dockerfile.latest-njs b/Dockerfile.latest-njs index bc12374d..ef9485a3 100644 --- a/Dockerfile.latest-njs +++ b/Dockerfile.latest-njs @@ -6,7 +6,9 @@ FROM nginx-s3-gateway RUN set -eux \ export DEBIAN_FRONTEND=noninteractive; \ apt-get update -qq; \ - apt-get install --no-install-recommends --no-install-suggests --yes make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev libpcre2-posix2 libxml2-dev libxslt1-dev zlib1g-dev; \ + apt-get install --no-install-recommends --no-install-suggests --yes \ + make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev \ + libpcre2-posix3 libxml2-dev libxslt1-dev zlib1g-dev; \ mkdir -p /tmp/nginx /tmp/njs-latest; \ curl --retry 6 --location "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ | gunzip | tar --extract --strip-components=1 --directory /tmp/nginx; \ @@ -19,8 +21,6 @@ RUN set -eux \ cd /tmp/nginx; \ ./configure \ --add-dynamic-module=/tmp/njs-latest/nginx \ - --without-http_gzip_module \ - --without-http_rewrite_module \ --prefix=/etc/nginx \ --sbin-path=/usr/sbin/nginx \ --modules-path=/usr/lib/nginx/modules \ @@ -36,9 +36,28 @@ RUN set -eux \ --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ --user=nginx --group=nginx --with-compat --with-file-aio \ --with-stream \ - --with-mail \ --with-threads \ --with-compat \ + --with-file-aio \ + --with-threads \ + --with-http_addition_module \ + --with-http_auth_request_module \ + --with-http_dav_module \ + --with-http_flv_module \ + --with-http_gunzip_module \ + --with-http_gzip_static_module \ + --with-http_mp4_module \ + --with-http_random_index_module \ + --with-http_realip_module \ + --with-http_secure_link_module \ + --with-http_slice_module \ + --with-http_ssl_module \ + --with-http_stub_status_module \ + --with-http_sub_module \ + --with-http_v2_module \ + --with-stream_realip_module \ + --with-stream_ssl_module \ + --with-stream_ssl_preread_module \ --with-cc-opt="-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-${NGINX_VERSION}/debian/debuild-base/nginx-${NGINX_VERSION}=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC"; \ make -j "$(nproc)"; \ cp objs/ngx_stream_js_module.so /usr/lib/nginx/modules; \ From 148609ff63b8280ebd1b8c809d6a316131aefaa8 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Fri, 15 Sep 2023 10:47:42 -0700 Subject: [PATCH 6/6] remove debug changes --- Dockerfile.debug | 70 ------------------------------------------- Dockerfile.latest-njs | 27 +++-------------- test.sh | 36 +++++++++------------- 3 files changed, 18 insertions(+), 115 deletions(-) delete mode 100644 Dockerfile.debug diff --git a/Dockerfile.debug b/Dockerfile.debug deleted file mode 100644 index 20f42e49..00000000 --- a/Dockerfile.debug +++ /dev/null @@ -1,70 +0,0 @@ -# This container image removes the existing njs package from the inherited image -# (which could be OSS NGINX or NGINX Plus), builds njs from the latest -# source, and installs it. -FROM nginx-s3-gateway - -RUN set -eux \ - export DEBIAN_FRONTEND=noninteractive; \ - apt-get update -qq; \ - apt-get install --no-install-recommends --no-install-suggests --yes \ - make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev \ - libpcre2-posix3 libxml2-dev libxslt1-dev zlib1g-dev; \ - mkdir -p /tmp/nginx /tmp/njs-latest; \ - curl --retry 6 --location "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ - | gunzip | tar --extract --strip-components=1 --directory /tmp/nginx; \ - curl --retry 6 --location 'https://hg.nginx.org/njs/archive/tip.tar.gz' \ - | gunzip | tar --extract --strip-components=1 --directory /tmp/njs-latest; \ - cd /tmp/njs-latest; \ - ./configure; \ - make -j "$(nproc)"; \ - cp build/njs /usr/bin/njs; \ - cd /tmp/nginx; \ - ./configure \ - --add-dynamic-module=/tmp/njs-latest/nginx \ - --prefix=/etc/nginx \ - --sbin-path=/usr/sbin/nginx \ - --modules-path=/usr/lib/nginx/modules \ - --conf-path=/etc/nginx/nginx.conf \ - --error-log-path=/var/log/nginx/error.log \ - --http-log-path=/var/log/nginx/access.log \ - --pid-path=/var/run/nginx.pid \ - --lock-path=/var/run/nginx.lock \ - --http-client-body-temp-path=/var/cache/nginx/client_temp \ - --http-proxy-temp-path=/var/cache/nginx/proxy_temp \ - --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \ - --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \ - --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ - --user=nginx --group=nginx --with-compat --with-file-aio \ - --with-debug \ - --with-stream \ - --with-threads \ - --with-compat \ - --with-file-aio \ - --with-threads \ - --with-http_addition_module \ - --with-http_auth_request_module \ - --with-http_dav_module \ - --with-http_flv_module \ - --with-http_gunzip_module \ - --with-http_gzip_static_module \ - --with-http_mp4_module \ - --with-http_random_index_module \ - --with-http_realip_module \ - --with-http_secure_link_module \ - --with-http_slice_module \ - --with-http_ssl_module \ - --with-http_stub_status_module \ - --with-http_sub_module \ - --with-http_v2_module \ - --with-stream_realip_module \ - --with-stream_ssl_module \ - --with-stream_ssl_preread_module \ - --with-cc-opt="-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-${NGINX_VERSION}/debian/debuild-base/nginx-${NGINX_VERSION}=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC"; \ - make -j "$(nproc)"; \ - cp objs/ngx_stream_js_module.so /usr/lib/nginx/modules; \ - cp objs/ngx_http_js_module.so /usr/lib/nginx/modules; \ - make install ; \ - apt-get purge --yes --auto-remove make gcc libc6-dev expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev zlib1g-dev; \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* diff --git a/Dockerfile.latest-njs b/Dockerfile.latest-njs index ef9485a3..bc12374d 100644 --- a/Dockerfile.latest-njs +++ b/Dockerfile.latest-njs @@ -6,9 +6,7 @@ FROM nginx-s3-gateway RUN set -eux \ export DEBIAN_FRONTEND=noninteractive; \ apt-get update -qq; \ - apt-get install --no-install-recommends --no-install-suggests --yes \ - make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev \ - libpcre2-posix3 libxml2-dev libxslt1-dev zlib1g-dev; \ + apt-get install --no-install-recommends --no-install-suggests --yes make gcc libc6-dev curl expect libpcre2-dev libpcre3-dev libedit-dev libreadline-dev libssl-dev libpcre2-posix2 libxml2-dev libxslt1-dev zlib1g-dev; \ mkdir -p /tmp/nginx /tmp/njs-latest; \ curl --retry 6 --location "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ | gunzip | tar --extract --strip-components=1 --directory /tmp/nginx; \ @@ -21,6 +19,8 @@ RUN set -eux \ cd /tmp/nginx; \ ./configure \ --add-dynamic-module=/tmp/njs-latest/nginx \ + --without-http_gzip_module \ + --without-http_rewrite_module \ --prefix=/etc/nginx \ --sbin-path=/usr/sbin/nginx \ --modules-path=/usr/lib/nginx/modules \ @@ -36,28 +36,9 @@ RUN set -eux \ --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ --user=nginx --group=nginx --with-compat --with-file-aio \ --with-stream \ + --with-mail \ --with-threads \ --with-compat \ - --with-file-aio \ - --with-threads \ - --with-http_addition_module \ - --with-http_auth_request_module \ - --with-http_dav_module \ - --with-http_flv_module \ - --with-http_gunzip_module \ - --with-http_gzip_static_module \ - --with-http_mp4_module \ - --with-http_random_index_module \ - --with-http_realip_module \ - --with-http_secure_link_module \ - --with-http_slice_module \ - --with-http_ssl_module \ - --with-http_stub_status_module \ - --with-http_sub_module \ - --with-http_v2_module \ - --with-stream_realip_module \ - --with-stream_ssl_module \ - --with-stream_ssl_preread_module \ --with-cc-opt="-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-${NGINX_VERSION}/debian/debuild-base/nginx-${NGINX_VERSION}=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC"; \ make -j "$(nproc)"; \ cp objs/ngx_stream_js_module.so /usr/lib/nginx/modules; \ diff --git a/test.sh b/test.sh index 14f9674f..d2be720e 100755 --- a/test.sh +++ b/test.sh @@ -66,25 +66,21 @@ e() { >&2 echo "$1" } -usage() { e "Usage: $0 [--latest-njs ] [--latest-njs-debug ] [--unprivileged ] [--type ]" 1>&2; exit 1; } +usage() { e "Usage: $0 [--latest-njs ] [--unprivileged ] [--type " 1>&2; exit 1; } for arg in "$@"; do shift case "$arg" in - '--help') set -- "$@" '-h' ;; - '--latest-njs') set -- "$@" '-j' ;; - '--latest-njs-debug') set -- "$@" '-d' ;; - '--unprivileged') set -- "$@" '-u' ;; - '--type') set -- "$@" '-t' ;; - *) set -- "$@" "$arg" ;; + '--help') set -- "$@" '-h' ;; + '--latest-njs') set -- "$@" '-j' ;; + '--unprivileged') set -- "$@" '-u' ;; + '--type') set -- "$@" '-t' ;; + *) set -- "$@" "$arg" ;; esac done -while getopts "hdjut:" arg; do +while getopts "hjut:" arg; do case "${arg}" in - d) - njs_latest_debug="1" - ;; j) njs_latest="1" ;; @@ -113,10 +109,10 @@ else startup_message="Starting NGINX ${nginx_type}" fi -if [ -z "${njs_latest}" ] && [ -z "${njs_latest_debug}" ]; then +if [ -z "${njs_latest}" ]; then njs_latest="0" startup_message="${startup_message} with the release NJS module (default)" -elif [[ ${njs_latest} -eq 1 ]] || [[ ${njs_latest_debug} -eq 1 ]]; then +elif [ ${njs_latest} -eq 1 ]; then startup_message="${startup_message} with the latest NJS module" else startup_message="${startup_message} with the release NJS module" @@ -315,11 +311,7 @@ else --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} . fi -if [ ${njs_latest_debug} -eq 1 ]; then - p "Layering in latest NJS build and nginx with debug logging" - docker build -f Dockerfile.debug \ - --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} . -elif [ ${njs_latest} -eq 1 ]; then +if [ ${njs_latest} -eq 1 ]; then p "Layering in latest NJS build" docker build -f Dockerfile.latest-njs \ --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} . @@ -419,13 +411,13 @@ integration_test 2 1 0 0 compose stop nginx-s3-gateway # Restart with new config -p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index" -integration_test 2 1 1 1 +p "Testing API with AWS Signature V2 and static site on" +integration_test 2 0 1 0 compose stop nginx-s3-gateway # Restart with new config -p "Testing API with AWS Signature V2 and static site on" -integration_test 2 0 1 0 +p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index" +integration_test 2 1 1 1 compose stop nginx-s3-gateway # Restart with new config