Feature/shadding

README.md

@@ -2,8 +2,8 @@
 
 |Branch|Status|
 |---|---|
-|master|[![Build status](https://ci.appveyor.com/api/projects/status/ebphtfegnposba6w?svg=true)](https://ci.appveyor.com/project/appsvc/azure-functions-java-worker?branch=master)|
-|dev|[![Build status](https://ci.appveyor.com/api/projects/status/ebphtfegnposba6w?svg=true)](https://ci.appveyor.com/project/appsvc/azure-functions-java-worker?branch=dev)|
+|master|[![Build status](https://dev.azure.com/azfunc/Azure%20Functions/_apis/build/status/Azure.azure-functions-java-worker?branchName=master)](https://dev.azure.com/azfunc/Azure%20Functions/_build/latest?definitionId=20&branchName=master)|
+|dev|[![Build status](https://dev.azure.com/azfunc/Azure%20Functions/_apis/build/status/Azure.azure-functions-java-worker?branchName=dev)](https://dev.azure.com/azfunc/Azure%20Functions/_build/latest?definitionId=20&branchName=dev)|
 
 # Contributing
 
@@ -104,6 +104,10 @@ Simply using the following command to do so (if there are dependency errors, run
 mvn javadoc:javadoc
 ```
 
+# Development Notes
+
+Java worker now shades all its jars, to introduce any new jars it is required by the developers to add a section in the pom file to relocate it.
+
 # Coding Convention
 
 ## Version Management

appveyor.yml

@@ -30,7 +30,7 @@ build_script:
        Write-Host “Found git tag."
       } 
       else {
-       $buildNumber = "1.6.2-$env:APPVEYOR_BUILD_NUMBER"
+       $buildNumber = "1.7.1-$env:APPVEYOR_BUILD_NUMBER"
        Write-Host “git tag not found. Setting package suffix to '$buildNumber'"
       }     
       .\package.ps1 -buildNumber $buildNumber

azure-pipelines.yml

@@ -19,10 +19,10 @@ steps:
        Write-Host “Found git tag."
       } 
       else {
-       $buildNumber = "1.6.2-$(Build.BuildId)"
+       $buildNumber = "1.7.1-$(Build.BuildId)"
        Write-Host “git tag not found. Setting package suffix to '$buildNumber'"
       }     
-      .\package.ps1 -buildNumber $buildNumber
+      .\package-pipeline.ps1 -buildNumber $buildNumber
   displayName: 'Executing build script'
 - task: CopyFiles@2
   inputs:
@@ -49,6 +49,22 @@ steps:
     SBQueueName: $(SBQueueName)
   displayName: 'Build & Run tests for java 8'
   continueOnError: false
+- pwsh: |
+    .\build-run-tests-pipeline.ps1
+  env:
+    FUNCTIONS_WORKER_JAVA_LOAD_APP_LIBS: 'True'
+    AzureWebJobsStorage: $(AzureWebJobsStorage)
+    AzureWebJobsCosmosDBConnectionString: $(AzureWebJobsCosmosDBConnectionString)
+    AzureWebJobsServiceBus: $(AzureWebJobsServiceBus)
+    AzureWebJobsEventHubReceiver: $(AzureWebJobsEventHubReceiver)
+    AzureWebJobsEventHubSender: $(AzureWebJobsEventHubSender)
+    AzureWebJobsEventHubPath: $(AzureWebJobsEventHubPath)
+    SBTopicName: $(SBTopicName)
+    SBTopicSubName: $(SBTopicSubName)
+    CosmosDBDatabaseName: $(CosmosDBDatabaseName)
+    SBQueueName: $(SBQueueName)
+  displayName: 'Build & Run tests for java 8 Customer jar loaded first'
+  continueOnError: false
 - pwsh: |
     .\build-run-tests-pipeline.ps1
   env:

e2e-nightly-cli-azure-pipelines.yml

@@ -19,7 +19,7 @@ steps:
        Write-Host “Found git tag."
       } 
       else {
-       $buildNumber = "1.6.2-$(Build.BuildId)"
+       $buildNumber = "1.7.1-$(Build.BuildId)"
        Write-Host “git tag not found. Setting package suffix to '$buildNumber'"
       }     
       .\package.ps1 -buildNumber $buildNumber

endtoendtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/HttpEndToEndTests.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System;
 using System.Net;
 using System.Threading.Tasks;
 using Xunit;
@@ -33,5 +34,26 @@ public async Task HttpTrigger_ReturnsCustomCode()
         {
             Assert.True(await Utilities.InvokeHttpTrigger("HttpTriggerCustomCode", "?&name=Test",  HttpStatusCode.OK, "Test", 209));
         }
+
+        [Fact]
+        public async Task HttpTriggerJavaClassLoader()
+        {
+            // The e2e project has newer jars than the one we use in the worker.
+            // The purpose of this test will be called for three scenarios:
+            // 1. Java 11 -- Client code takes presence. -- works fine.
+            // 2. Java 8 with no Application settings, worker lib jars takes presence -- throw exception
+            // 3. Java 8 with with Application settings, worker lib jars takes presence -- works fine.
+
+            String value = Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_JAVA_LOAD_APP_LIBS");
+            String java_home = Environment.GetEnvironmentVariable("JAVA_HOME");
+            if (java_home.Contains("zulu-11") || (value != null && value.ToLower().Equals("true")))
+            {
+                Assert.True(await Utilities.InvokeHttpTrigger("HttpTriggerJavaClassLoader", "?&name=Test", HttpStatusCode.OK, "Test"));
+            }
+            else
+            {
+                Assert.True(await Utilities.InvokeHttpTrigger("HttpTriggerJavaClassLoader", "?&name=Test", HttpStatusCode.InternalServerError, ""));
+            }
+        }
     }
 }

endtoendtests/src/main/java/com/microsoft/azure/functions/endtoend/HttpTriggerTests.java

@@ -46,12 +46,6 @@ public HttpResponseMessage HttpTriggerJava(
             }
         }
 
-        Gson a = new Gson();
-
-//        if(!SystemUtils.IS_JAVA_15) {
-//            context.getLogger().info("Java version not 15");
-//        }
-
         get("https://httpstat.us/200");
 
         if (name == null ) {
@@ -108,4 +102,28 @@ public HttpResponseMessage HttpTriggerCustomCode(
             return request.createResponseBuilder(HttpStatusType.custom(209)).body("Hello, " + name).build();
         }
     }
+
+    @FunctionName("HttpTriggerJavaClassLoader")
+    public HttpResponseMessage HttpTriggerJavaClassLoader(
+            @HttpTrigger(name = "req", methods = {HttpMethod.GET, HttpMethod.POST}, authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage<Optional<String>> request,
+            final ExecutionContext context
+    ) throws Exception {
+        context.getLogger().info("Java HTTP trigger processed a request.");
+
+        // Parse query parameters
+        String query = request.getQueryParameters().get("name");
+        String name = request.getBody().orElse(query);
+
+        //Make sure there is no class not found excption.
+        new Gson();
+
+        if(!SystemUtils.IS_JAVA_15) {
+            context.getLogger().info("Java version not 15");
+        }
+
+        if (name == null ) {
+            return request.createResponseBuilder(HttpStatus.BAD_REQUEST).body("Please pass a name on the query string or in the request body").build();
+        }
+        return request.createResponseBuilder(HttpStatus.OK).body("Hello, " + name).build();
+    }
 }

package-pipeline.ps1

@@ -0,0 +1,29 @@
+param (
+  [string]$buildNumber = $env:APPVEYOR_BUILD_NUMBER
+)
+
+# A function that checks exit codes and fails script if an error is found 
+function StopOnFailedExecution {
+  if ($LastExitCode) 
+  { 
+    exit $LastExitCode 
+  }
+}
+Write-Host "Building azure-functions-java-worker" 
+cmd.exe /c '.\mvnBuild.bat'
+StopOnFailedExecution
+
+Write-Host "Creating nuget package Microsoft.Azure.Functions.JavaWorker" 
+Write-Host "buildNumber: " $buildNumber
+Get-Command nuget
+StopOnFailedExecution
+remove-item pkg -Recurse -ErrorAction Ignore
+mkdir pkg
+Get-ChildItem -Path .\target\* -Include 'azure*' -Exclude '*shaded.jar','*tests.jar' | %{ Copy-Item $_.FullName .\pkg\azure-functions-java-worker.jar }
+StopOnFailedExecution
+copy-item ./worker.config.json pkg
+copy-item ./tools/AzureFunctionsJavaWorker.nuspec pkg/
+Copy-Item ".\lib_worker_1.6.2" pkg\lib -Recurse
+set-location pkg
+nuget pack -Properties version=$buildNumber
+set-location ..

package.ps1

@@ -23,6 +23,7 @@ Get-ChildItem -Path .\target\* -Include 'azure*' -Exclude '*shaded.jar' | %{ Cop
 StopOnFailedExecution
 copy-item ./worker.config.json pkg
 copy-item ./tools/AzureFunctionsJavaWorker.nuspec pkg/
+Copy-Item ".\lib_worker_1.6.2" pkg\lib -Recurse
 set-location pkg
 nuget pack -Properties version=$buildNumber
 set-location ..

pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.microsoft.azure.functions</groupId>
 	<artifactId>azure-functions-java-worker</artifactId>
-	<version>1.6.2</version>
+	<version>1.7.1</version>
 	<packaging>jar</packaging>
 	<parent>
 		<groupId>com.microsoft.maven</groupId>
@@ -174,7 +174,7 @@
 			</plugin>
 			<plugin>
 				<artifactId>maven-shade-plugin</artifactId>
-				<version>3.2.0</version>
+				<version>3.2.4</version>
 				<executions>
 					<execution>
 						<phase>package</phase>
@@ -184,14 +184,15 @@
 						<configuration>
 							<transformers>
 								<transformer
-									implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+										implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
 									<mainClass>com.microsoft.azure.functions.worker.Application</mainClass>
 									<manifestEntries>
 										<Implementation-Title>${project.name}</Implementation-Title>
 										<Implementation-Version>${project.version}</Implementation-Version>
 										<Implementation-Vendor>${project.organization.name}</Implementation-Vendor>
 									</manifestEntries>
 								</transformer>
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
 							</transformers>
 							<filters>
 								<filter>
@@ -203,6 +204,56 @@
 									</excludes>
 								</filter>
 							</filters>
+							<relocations>
+								<relocation combine.children="append">
+									<pattern>com.google</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.com.google</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>com.sun</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.com.sun</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>android</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.android</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>google</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.google</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>identity</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.identity</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>shared</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.shared</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>org</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.org</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>io.netty</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.io.netty</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>io.perfmark</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.io.perfmark</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>io.grpc</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.io.grpc</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>javax.annotation</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.javax.annotation</shadedPattern>
+								</relocation>
+								<relocation combine.children="append">
+									<pattern>io.opencensus</pattern>
+									<shadedPattern>com.microsoft.azure.functions.shaded.io.opencensus</shadedPattern>
+								</relocation>
+							</relocations>
 						</configuration>
 					</execution>
 				</executions>

setup-tests-pipeline.ps1

@@ -36,4 +36,4 @@ if(!$skipCliDownload)
 Write-Host "Copying azure-functions-java-worker to  Functions Host workers directory...."
 Get-ChildItem -Path .\target\* -Include 'azure*' -Exclude '*shaded.jar','*tests.jar' | %{ Copy-Item $_.FullName ".\Azure.Functions.Cli\workers\java\azure-functions-java-worker.jar" }
 Copy-Item ".\worker.config.json" ".\Azure.Functions.Cli\workers\java"
-
+Copy-Item ".\lib_worker_1.6.2" ".\Azure.Functions.Cli\workers\java\lib" -Recurse

setup-tests.ps1

@@ -44,6 +44,7 @@ if(!$skipCliDownload)
 Write-Host "Copying azure-functions-java-worker to  Functions Host workers directory...."
 Get-ChildItem -Path .\target\* -Include 'azure*' -Exclude '*shaded.jar','*tests.jar' | %{ Copy-Item $_.FullName ".\Azure.Functions.Cli\workers\java\azure-functions-java-worker.jar" }
 Copy-Item ".\worker.config.json" ".\Azure.Functions.Cli\workers\java"
+Copy-Item ".\lib_worker_1.6.2" ".\Azure.Functions.Cli\workers\java\lib" -Recurse
 
 Write-Host "Building endtoendtests...."
 $Env:Path = $Env:Path+";$currDir\Azure.Functions.Cli"

src/main/java/com/microsoft/azure/functions/worker/Constants.java

@@ -6,4 +6,6 @@
 public final class Constants {
     private Constants(){}
     public final static String TRIGGER_METADATA_DOLLAR_REQUEST_KEY = "$request";
+    public final static String FUNCTIONS_WORKER_DIRECTORY = "FUNCTIONS_WORKER_DIRECTORY";
+    public final static String FUNCTIONS_WORKER_JAVA_LOAD_APP_LIBS = "FUNCTIONS_WORKER_JAVA_LOAD_APP_LIBS";
 }

src/main/java/com/microsoft/azure/functions/worker/Helper.java

@@ -0,0 +1,9 @@
+package com.microsoft.azure.functions.worker;
+
+public class Helper {
+
+    public static boolean isLoadAppLibsFirst() {
+        String javaReverseLibLoading = System.getenv(Constants.FUNCTIONS_WORKER_JAVA_LOAD_APP_LIBS);
+        return Util.isTrue(javaReverseLibLoading);
+    }
+}

src/main/java/com/microsoft/azure/functions/worker/Util.java

@@ -0,0 +1,10 @@
+package com.microsoft.azure.functions.worker;
+
+public class Util {
+    public static boolean isTrue(String value) {
+        if(value != null && (value.toLowerCase().equals("true") || value.toLowerCase().equals("1"))) {
+            return true;
+        }
+        return false;
+    }
+}

src/main/java/com/microsoft/azure/functions/worker/broker/JavaFunctionBroker.java

@@ -1,17 +1,22 @@
 package com.microsoft.azure.functions.worker.broker;
 
 import java.io.File;
+import java.io.FileFilter;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URL;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.microsoft.azure.functions.rpc.messages.*;
 import com.microsoft.azure.functions.worker.Constants;
+import com.microsoft.azure.functions.worker.Helper;
+import com.microsoft.azure.functions.worker.Util;
 import com.microsoft.azure.functions.worker.binding.BindingDataStore;
 import com.microsoft.azure.functions.worker.description.FunctionMethodDescriptor;
 import com.microsoft.azure.functions.worker.reflect.ClassLoaderProvider;
 
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 
@@ -87,14 +92,46 @@ private void addSearchPathsToClassLoader(FunctionMethodDescriptor function) thro
 		function.getLibDirectory().ifPresent(d -> registerWithClassLoaderProvider(d));
 	}
 
-	private void registerWithClassLoaderProvider(File libDirectory) {
+	void registerWithClassLoaderProvider(File libDirectory) {
 		try {
-			classLoaderProvider.addDirectory(libDirectory);
+			if(SystemUtils.IS_JAVA_1_8) {
+				String workerLibPath = System.getenv(Constants.FUNCTIONS_WORKER_DIRECTORY) + "/lib";
+				File workerLib = new File(workerLibPath);
+				verifyLibrariesExist (workerLib, workerLibPath);
+
+				if(Helper.isLoadAppLibsFirst()) {
+					// load client app jars first.
+					classLoaderProvider.addDirectory(libDirectory);
+					classLoaderProvider.addDirectory(workerLib);
+				} else {
+					// Default load worker jars first.
+					classLoaderProvider.addDirectory(workerLib);
+					classLoaderProvider.addDirectory(libDirectory);
+				}
+			} else {
+				classLoaderProvider.addDirectory(libDirectory);
+			}
 		} catch (Exception ex) {
 			ExceptionUtils.rethrow(ex);
 		}
 	}
 
+	void verifyLibrariesExist (File workerLib, String workerLibPath) throws FileNotFoundException{
+		if(!workerLib.exists()) {
+			throw new FileNotFoundException("Error loading worker jars, from path:  " + workerLibPath);
+		} else {
+			File[] jarFiles = workerLib.listFiles(new FileFilter() {
+				@Override
+				public boolean accept(File file) {
+					return file.isFile() && file.getName().endsWith(".jar");
+				}
+			});
+			if(jarFiles.length == 0) {
+				throw new FileNotFoundException("Error loading worker jars, from path:  " + workerLibPath + ". Jars size is zero");
+			}
+		}
+	}
+
 	private final Map<String, ImmutablePair<String, JavaMethodExecutor>> methods;
 	private final ClassLoaderProvider classLoaderProvider;
 }