[Xamarin.Android.Build.Tasks] use System.Reflection.Metadata in <ResolveAssemblies/>

Context: https://github.com/dotnet/corefx/tree/master/src/System.Reflection.Metadata/src/System/Reflection/Metadata
Context: https://github.com/jonathanpeppers/Benchmarks

There is a new System.Reflection.Metadata library from corefx for
reading .NET assemblies. It is a bit more performant than Mono.Cecil
because it is a different library with different opinions.

Some notes about System.Reflection.Metadata:

- SRM has a forward "reader" style API
- SRM uses lots of structs, and you have to do an additional call to
lookup strings generally.
- SRM, as far as I have seen, doesn't have APIs to modify and write
out new assemblies.
- SRM only supports "portable" pdb files.
- SRM is not well documented yet. To discover usage, I read source
  code and/or unit tests.

From my benchmark above, it seems that SRM is 10x faster on
Windows/.NET framework and 5x faster on macOS/Mono.

So it makes sense for use to use SRM when reading assemblies (and we
don't need symbols), and continue with Mono.Cecil for the linker and
other things that modify assemblies.

There are a few places we can take advantage of SRM, but the simplest
with a reasonable impact was `ResolveAssemblies`:

    Before:
    320 ms  ResolveAssemblies                          1 calls
    After:
    112 ms  ResolveAssemblies                          1 calls

So a ~200ms savings on this MSBuild task, which runs on *every* build.
This was the Xamarin.Forms test project in this repo: a build with no
changes.

~~ Changes ~~

- Added a `MetadataResolver` type, as a way to cache `PEReader`
  instances. This is a comparable drop-in replacement for
  `DirectoryAssemblyResolver`.
- `MonoAndroidHelper.IsReferenceAssembly` now uses
  `System.Reflection.Metadata` instead of `Mono.Cecil`. This is used
  in a few other MSBuild tasks.
- A `MetadataExtensions` provides an extension method to simplify
  getting the full name of a custom attribute. We can add more here as
  needed.
- Had to adjust the filename reported for XA2002, should optionally
  call `Path.GetFileNameWithoutExtension` if the name ends with
  `.dll`.

The resulting code *should* be the same, except we are using SRM over
Mono.Cecil.

~~ Other changes ~~

[xabuild.exe] remove SRM reference

This appears to fix the build on macOS, we had this workaround from a
mono bump in the past. Since xabuild has its own version of
System.Reflection.Metadata that was already loaded, we weren't loading
the one we are using in XA's MSBuild tasks.

Things appear to work without the reference now.

~~ Downstream ~~

We will need to add the following assemblies to the installer:

- `System.Reflection.Metadata.dll`
- `System.Collections.Immutable.dll`

Author: jonathanpeppers

src/Xamarin.Android.Build.Tasks/Tasks/ResolveAssemblies.cs

@@ -1,21 +1,18 @@
 // Copyright (C) 2011, Xamarin Inc.
 // Copyright (C) 2010, Novell Inc.
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
-using Mono.Cecil;
 using MonoDroid.Tuner;
+using NuGet.Frameworks;
+using NuGet.ProjectModel;
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Reflection.Metadata;
 using System.Text;
 using Xamarin.Android.Tools;
-using NuGet.Common;
-using NuGet.Frameworks;
-using NuGet.ProjectModel;
-
-using Java.Interop.Tools.Cecil;
 
 namespace Xamarin.Android.Tasks
 {
@@ -62,7 +59,7 @@ public override bool Execute ()
 			Yield ();
 			try {
 				System.Threading.Tasks.Task.Run (() => {
-					using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: false)) {
+					using (var resolver = new MetadataResolver ()) {
 						Execute (resolver);
 					}
 				}, Token).ContinueWith (Complete);
@@ -72,14 +69,13 @@ public override bool Execute ()
 			}
 		}
 
-		void Execute (DirectoryAssemblyResolver resolver)
+		void Execute (MetadataResolver resolver)
 		{
-			foreach (var dir in ReferenceAssembliesDirectory.Split (new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
-				resolver.SearchDirectories.Add (dir);
-
-			var assemblies = new Dictionary<string, ITaskItem> ();
+			foreach (var dir in ReferenceAssembliesDirectory.Split (new char [] { ';' }, StringSplitOptions.RemoveEmptyEntries))
+				resolver.AddSearchDirectory (dir);
 
-			var topAssemblyReferences = new List<AssemblyDefinition> ();
+			var assemblies = new Dictionary<string, ITaskItem> (Assemblies.Length);
+			var topAssemblyReferences = new List<string> (Assemblies.Length);
 			var logger = new NuGetLogger((s) => {
 				LogDebugMessage ("{0}", s);
 			});
@@ -92,32 +88,28 @@ void Execute (DirectoryAssemblyResolver resolver)
 			try {
 				foreach (var assembly in Assemblies) {
 					var assembly_path = Path.GetDirectoryName (assembly.ItemSpec);
-
-					if (!resolver.SearchDirectories.Contains (assembly_path))
-						resolver.SearchDirectories.Add (assembly_path);
+					resolver.AddSearchDirectory (assembly_path);
 
 					// Add each user assembly and all referenced assemblies (recursive)
-					var assemblyDef = resolver.Load (assembly.ItemSpec);
-					if (assemblyDef == null)
-						throw new InvalidOperationException ("Failed to load assembly " + assembly.ItemSpec);
-					if (MonoAndroidHelper.IsReferenceAssembly (assemblyDef)) {
+					string resolved_assembly = resolver.Resolve (assembly.ItemSpec);
+					if (MonoAndroidHelper.IsReferenceAssembly (resolved_assembly)) {
 						// Resolve "runtime" library
-						var asmFullPath = Path.GetFullPath (assembly.ItemSpec);
 						if (lockFile != null)
-							assemblyDef = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, resolver, asmFullPath);
-						if (lockFile == null || assemblyDef == null) {
-							LogCodedWarning ("XA0107", asmFullPath, 0, "Ignoring {0} as it is a Reference Assembly", asmFullPath);
+							resolved_assembly = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, assembly.ItemSpec);
+						if (lockFile == null || resolved_assembly == null) {
+							LogCodedWarning ("XA0107", resolved_assembly, 0, "Ignoring {0} as it is a Reference Assembly", resolved_assembly);
 							continue;
 						}
 					}
-					topAssemblyReferences.Add (assemblyDef);
+					topAssemblyReferences.Add (resolved_assembly);
 					var taskItem = new TaskItem (assembly) {
-						ItemSpec = Path.GetFullPath (assemblyDef.MainModule.FileName),
+						ItemSpec = Path.GetFullPath (resolved_assembly),
 					};
 					if (string.IsNullOrEmpty (taskItem.GetMetadata ("ReferenceAssembly"))) {
 						taskItem.SetMetadata ("ReferenceAssembly", taskItem.ItemSpec);
 					}
-					assemblies [assemblyDef.Name.Name] = taskItem;
+					string assemblyName = Path.GetFileNameWithoutExtension (resolved_assembly);
+					assemblies [assemblyName] = taskItem;
 				}
 			} catch (Exception ex) {
 				LogError ("Exception while loading assemblies: {0}", ex);
@@ -171,7 +163,7 @@ void Execute (DirectoryAssemblyResolver resolver)
 		readonly Dictionary<string, int> api_levels = new Dictionary<string, int> ();
 		int indent = 2;
 
-		AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, DirectoryAssemblyResolver resolver, string assemblyPath)
+		string ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, string assemblyPath)
 		{
 			if (string.IsNullOrEmpty(TargetMoniker)) 
 				return null;
@@ -200,16 +192,16 @@ AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile
 				path = Path.Combine (folder.Path, libraryPath.Path, runtime.Path).Replace('/', Path.DirectorySeparatorChar);
 				if (!File.Exists (path))
 					continue;
-				LogDebugMessage ($"Attempting to load {path}");
-				return resolver.Load (path, forceLoad: true);
+				return path;
 			}
 			return null;
 		}
 
-		void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies, AssemblyDefinition assembly, List<string> resolutionPath)
+		void AddAssemblyReferences (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies, string assemblyPath, List<string> resolutionPath)
 		{
-			var assemblyName = assembly.Name.Name;
-			var fullPath = Path.GetFullPath (assembly.MainModule.FileName);
+			var reader = resolver.GetAssemblyReader (assemblyPath);
+			var assembly = reader.GetAssemblyDefinition ();
+			var assemblyName = reader.GetString (assembly.Name);
 
 			// Don't repeat assemblies we've already done
 			bool topLevel = resolutionPath == null;
@@ -219,22 +211,23 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
 			if (resolutionPath == null)
 				resolutionPath = new List<string>();
 
-			CheckAssemblyAttributes (assembly);
+			CheckAssemblyAttributes (assembly, reader);
 
-			LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assembly.Name);
-			resolutionPath.Add (assembly.Name.Name);
+			LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assemblyName);
+			resolutionPath.Add (assemblyName);
 			indent += 2;
 
 			// Add this assembly
 			if (!topLevel) {
-				assemblies [assemblyName] = CreateAssemblyTaskItem (fullPath);
+				assemblies [assemblyName] = CreateAssemblyTaskItem (Path.GetFullPath (assemblyPath));
 			}
 
 			// Recurse into each referenced assembly
-			foreach (AssemblyNameReference reference in assembly.MainModule.AssemblyReferences) {
-				AssemblyDefinition reference_assembly;
+			foreach (var handle in reader.AssemblyReferences) {
+				var reference = reader.GetAssemblyReference (handle);
+				string reference_assembly;
 				try {
-					reference_assembly = resolver.Resolve (reference);
+					reference_assembly = resolver.Resolve (reader.GetString (reference.Name));
 				} catch (FileNotFoundException ex) {
 					var references = new StringBuilder ();
 					for (int i = 0; i < resolutionPath.Count; i++) {
@@ -245,7 +238,10 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
 						references.Append ('`');
 					}
 
-					string missingAssembly = Path.GetFileNameWithoutExtension (ex.FileName);
+					string missingAssembly = ex.FileName;
+					if (missingAssembly.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
+						missingAssembly = Path.GetFileNameWithoutExtension (missingAssembly);
+					}
 					string message = $"Can not resolve reference: `{missingAssembly}`, referenced by {references}.";
 					if (MonoAndroidHelper.IsFrameworkAssembly (ex.FileName)) {
 						LogCodedError ("XA2002", $"{message} Perhaps it doesn't exist in the Mono for Android profile?");
@@ -261,25 +257,30 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
 			resolutionPath.RemoveAt (resolutionPath.Count - 1);
 		}
 
-		void CheckAssemblyAttributes (AssemblyDefinition assembly)
+		void CheckAssemblyAttributes (AssemblyDefinition assembly, MetadataReader reader)
 		{
-			foreach (var att in assembly.CustomAttributes) {
-				switch (att.AttributeType.FullName) {
+			foreach (var handle in assembly.GetCustomAttributes ()) {
+				var attribute = reader.GetCustomAttribute (handle);
+				switch (reader.GetCustomAttributeFullName (attribute)) {
 					case "Java.Interop.DoNotPackageAttribute": {
-							string file = (string)att.ConstructorArguments.First ().Value;
-							if (string.IsNullOrWhiteSpace (file))
-								LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.FullName);
-							do_not_package_atts.Add (Path.GetFileName (file));
+							var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
+							if (decoded.FixedArguments.Length > 0) {
+								string file = decoded.FixedArguments [0].Value?.ToString ();
+								if (string.IsNullOrWhiteSpace (file))
+									LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.GetAssemblyName ().FullName);
+								do_not_package_atts.Add (Path.GetFileName (file));
+							}
 						}
 						break;
 					case "System.Runtime.Versioning.TargetFrameworkAttribute": {
-							foreach (var p in att.ConstructorArguments) {
-								var value = p.Value.ToString ();
-								if (value.StartsWith ("MonoAndroid")) {
+							var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
+							foreach (var p in decoded.FixedArguments) {
+								var value = p.Value?.ToString ();
+								if (value != null && value.StartsWith ("MonoAndroid", StringComparison.Ordinal)) {
 									var values = value.Split ('=');
 									var apiLevel = MonoAndroidHelper.SupportedVersions.GetApiLevelFromFrameworkVersion (values [1]);
 									if (apiLevel != null) {
-										var assemblyName = assembly.Name.Name;
+										var assemblyName = reader.GetString (assembly.Name);
 										Log.LogDebugMessage ("{0}={1}", assemblyName, apiLevel);
 										api_levels [assemblyName] = apiLevel.Value;
 									}
@@ -305,7 +306,7 @@ static LinkModes ParseLinkMode (string linkmode)
 			return mode;
 		}
 
-		void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies)
+		void AddI18nAssemblies (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies)
 		{
 			var i18n = Linker.ParseI18nAssemblies (I18nAssemblies);
 			var link = ParseLinkMode (LinkMode);
@@ -332,10 +333,10 @@ void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, I
 				ResolveI18nAssembly (resolver, "I18N.West", assemblies);
 		}
 
-		void ResolveI18nAssembly (DirectoryAssemblyResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
+		void ResolveI18nAssembly (MetadataResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
 		{
-			var assembly = resolver.Resolve (AssemblyNameReference.Parse (name));
-			var assemblyFullPath = Path.GetFullPath (assembly.MainModule.FileName);
+			var assembly = resolver.Resolve (name);
+			var assemblyFullPath = Path.GetFullPath (assembly);
 			assemblies [name] = CreateAssemblyTaskItem (assemblyFullPath);
 		}
 

src/Xamarin.Android.Build.Tasks/Utilities/DummyCustomAttributeProvider.cs

@@ -0,0 +1,32 @@
+using System.Reflection.Metadata;
+
+namespace Xamarin.Android.Tasks
+{
+	/// <summary>
+	/// A helper type for System.Reflection.Metadata. Getting the value of custom attribute arguments is a bit convoluted, if you merely want the values.
+	///
+	/// This interface allows usage such as:
+	///		CustomAttribute attribute = reader.GetCustomAttribute (handle);
+	///		CustomAttributeValue<object> decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
+	/// </summary>
+	public class DummyCustomAttributeProvider : ICustomAttributeTypeProvider<object>
+	{
+		public static readonly DummyCustomAttributeProvider Instance = new DummyCustomAttributeProvider ();
+
+		public object GetPrimitiveType (PrimitiveTypeCode typeCode) => null;
+
+		public object GetSystemType () => null;
+
+		public object GetSZArrayType (object elementType) => null;
+
+		public object GetTypeFromDefinition (MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => null;
+
+		public object GetTypeFromReference (MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => null;
+
+		public object GetTypeFromSerializedName (string name) => null;
+
+		public PrimitiveTypeCode GetUnderlyingEnumType (object type) => default (PrimitiveTypeCode);
+
+		public bool IsSystemType (object type) => false;
+	}
+}

src/Xamarin.Android.Build.Tasks/Utilities/MetadataExtensions.cs

@@ -0,0 +1,21 @@
+using System.Reflection.Metadata;
+
+namespace Xamarin.Android.Tasks
+{
+	public static class MetadataExtensions
+	{
+		public static string GetCustomAttributeFullName (this MetadataReader reader, CustomAttribute attribute)
+		{
+			if (attribute.Constructor.Kind == HandleKind.MemberReference) {
+				var ctor = reader.GetMemberReference ((MemberReferenceHandle)attribute.Constructor);
+				var type = reader.GetTypeReference ((TypeReferenceHandle)ctor.Parent);
+				return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
+			} else if (attribute.Constructor.Kind == HandleKind.MethodDefinition) {
+				var ctor = reader.GetMethodDefinition ((MethodDefinitionHandle)attribute.Constructor);
+				var type = reader.GetTypeDefinition (ctor.GetDeclaringType ());
+				return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
+			}
+			return null;
+		}
+	}
+}

src/Xamarin.Android.Build.Tasks/Utilities/MetadataResolver.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+
+namespace Xamarin.Android.Tasks
+{
+	/// <summary>
+	/// A replacement for DirectoryAssemblyResolver, using System.Reflection.Metadata
+	/// </summary>
+	public class MetadataResolver : IDisposable
+	{
+		readonly Dictionary<string, PEReader> cache = new Dictionary<string, PEReader> ();
+		readonly List<string> searchDirectories = new List<string> ();
+
+		public MetadataReader GetAssemblyReader (string assemblyName)
+		{
+			var key = Path.GetFileNameWithoutExtension (assemblyName);
+			if (!cache.TryGetValue (key, out PEReader reader)) {
+				var assemblyPath = Resolve (assemblyName);
+				cache.Add (key, reader = new PEReader (File.OpenRead (assemblyPath)));
+			}
+			return reader.GetMetadataReader ();
+		}
+
+		public void AddSearchDirectory (string directory)
+		{
+			directory = Path.GetFullPath (directory);
+			if (!searchDirectories.Contains (directory))
+				searchDirectories.Add (directory);
+		}
+
+		public string Resolve (string assemblyName)
+		{
+			string assemblyPath = assemblyName;
+			if (!assemblyPath.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
+				assemblyPath += ".dll";
+			}
+			if (File.Exists (assemblyPath)) {
+				return assemblyPath;
+			}
+			foreach (var dir in searchDirectories) {
+				var path = Path.Combine (dir, assemblyPath);
+				if (File.Exists (path))
+					return path;
+			}
+
+			throw new FileNotFoundException ($"Could not load assembly '{assemblyName}'.", assemblyName);
+		}
+
+		public void Dispose ()
+		{
+			foreach (var provider in cache.Values) {
+				provider.Dispose ();
+			}
+			cache.Clear ();
+		}
+	}
+}