[DRAFT] Build binding projects on Windows instead of remotely. Fixes #16244.

dotnet/targets/Xamarin.Shared.Sdk.props

@@ -191,4 +191,10 @@
 		<!-- We're using our own native main function when using NativeAOT -->
 		<CustomNativeMain>true</CustomNativeMain>
 	</PropertyGroup>
+
+	<!-- Compute _SdkIsSimulator from the RuntimeIdentifier -->
+	<PropertyGroup>
+		<_SdkIsSimulator Condition="'$(RuntimeIdentifier)' != '' And '$(_SdkIsSimulator)' == ''">$(RuntimeIdentifier.Contains('simulator'))</_SdkIsSimulator>
+		<_SdkIsSimulator Condition="'$(RuntimeIdentifiers)' != '' And '$(_SdkIsSimulator)' == ''">$(RuntimeIdentifiers.Contains('simulator'))</_SdkIsSimulator>
+	</PropertyGroup>
 </Project>

msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx

@@ -1540,4 +1540,8 @@
             {0}: the path to an assembly
         </comment>
     </data>
+
+    <data name="E7120" xml:space="preserve">
+        <value>Can't process the native reference '{0}' on this platform because it is or contains a symlink.</value>
+    </data>
 </root>

msbuild/Xamarin.MacDev.Tasks/Decompress.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Reflection;
 
 using Microsoft.Build.Framework;
@@ -215,6 +216,133 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 			return rv;
 		}
 
+		/// <summary>
+		/// Compresses the specified resources (may be either files or directories) into a zip file.
+		///
+		/// Fails if:
+		/// * The resources is or contains a symlink and we're executing on Windows.
+		/// * The resources isn't found inside the zip file.
+		/// </summary>
+		/// <param name="log"></param>
+		/// <param name="zip">The zip to create</param>
+		/// <param name="resources">The files or directories to compress.</param>
+		/// <returns></returns>
+		public static bool TryCompress (TaskLoggingHelper log, string zip, IEnumerable<string> resources, bool overwrite, string workingDirectory, bool maxCompression = false)
+		{
+			// We use 'zip' to compress on !Windows, and System.IO.Compression to extract on Windows.
+			// This is because System.IO.Compression doesn't handle symlinks correctly, so we can only use
+			// it on Windows. It's also possible to set the XAMARIN_USE_SYSTEM_IO_COMPRESSION=1 environment
+			// variable to force using System.IO.Compression on !Windows, which is particularly useful when
+			// testing the System.IO.Compression implementation locally (with the caveat that if the resources
+			// to compress has symlinks, it may not work).
+
+			if (overwrite) {
+				if (File.Exists (zip)) {
+					log.LogMessage (MessageImportance.Low, "Replacing zip file {0} with {1}", zip, string.Join (", ", resources));
+					File.Delete (zip);
+				} else {
+					log.LogMessage (MessageImportance.Low, "Creating zip file {0} with {1}", zip, string.Join (", ", resources));
+				}
+			} else {
+				if (File.Exists (zip)) {
+					log.LogMessage (MessageImportance.Low, "Updating zip file {0} with {1}", zip, string.Join (", ", resources));
+				} else {
+					log.LogMessage (MessageImportance.Low, "Creating new zip file {0} with {1}", zip, string.Join (", ", resources));
+				}
+			}
+
+			var zipdir = Path.GetDirectoryName (zip);
+			if (!string.IsNullOrEmpty (zipdir))
+				Directory.CreateDirectory (zipdir);
+
+			bool rv;
+			if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
+				rv = TryCompressUsingSystemIOCompression (log, zip, resources, workingDirectory, maxCompression);
+			} else if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("XAMARIN_USE_SYSTEM_IO_COMPRESSION"))) {
+				rv = TryCompressUsingSystemIOCompression (log, zip, resources, workingDirectory, maxCompression);
+			} else {
+				rv = TryCompressUsingUnzip (log, zip, resources, workingDirectory, maxCompression);
+			}
+
+			return rv;
+		}
+
+		// Will add to an existing zip file (not replace)
+		static bool TryCompressUsingUnzip (TaskLoggingHelper log, string zip, IEnumerable<string> resources, string workingDirectory, bool maxCompression)
+		{
+			var zipArguments = new List<string> ();
+			if (maxCompression)
+				zipArguments.Add ("-9");
+			zipArguments.Add ("-r");
+			zipArguments.Add ("-y");
+			zipArguments.Add (zip);
+
+			foreach (var resource in resources) {
+				var fullPath = Path.GetFullPath (resource);
+				zipArguments.Add (Path.GetFileName (fullPath));
+			}
+			var rv = XamarinTask.ExecuteAsync (log, "zip", zipArguments, workingDirectory: workingDirectory).Result;
+			log.LogMessage (MessageImportance.Low, "Updated {0} with {1}: {2}", zip, string.Join (", ", resources), rv.ExitCode == 0);
+			return rv.ExitCode == 0;
+		}
+
+#if NET
+		const CompressionLevel SmallestCompressionLevel = CompressionLevel.SmallestSize;
+#else
+		const CompressionLevel SmallestCompressionLevel = CompressionLevel.Optimal;
+#endif
+
+		// Will add to an existing zip file (not replace)
+		static bool TryCompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, IEnumerable<string> resources, string workingDirectory, bool maxCompression)
+		{
+			var rv = true;
+
+			workingDirectory = Path.GetFullPath (workingDirectory);
+
+			var resourcePaths = resources.Select ((v) => Path.Combine (workingDirectory, v)).ToList ();
+			foreach (var resource in resourcePaths) {
+				if (!resource.StartsWith (workingDirectory, StringComparison.Ordinal))
+					throw new InvalidOperationException ($"The resource to compress '{resource}' must be inside the working directory '{workingDirectory}'");
+			}
+
+			using var archive = ZipFile.Open (zip, File.Exists (zip) ? ZipArchiveMode.Update : ZipArchiveMode.Create);
+
+			var rootDirLength = workingDirectory.Length;
+			foreach (var resource in resourcePaths) {
+				if (Directory.Exists (resource)) {
+					var entries = Directory.GetFileSystemEntries (resource, "*", SearchOption.AllDirectories);
+					var entriesWithZipName = entries.Select (v => new { Path = v, ZipName = v.Substring (rootDirLength) });
+					foreach (var entry in entriesWithZipName) {
+						if (Directory.Exists (entry.Path)) {
+							if (entries.Where (v => v.StartsWith (entry.Path, StringComparison.Ordinal)).Count () == 1) {
+								// this is a directory with no files inside, we need to create an entry with a trailing directory separator.
+								archive.CreateEntry (entry.ZipName + zipDirectorySeparator);
+							}
+						} else {
+							WriteFileToZip (log, archive, entry.Path, entry.ZipName, maxCompression);
+						}
+					}
+				} else if (File.Exists (resource)) {
+					var zipName = resource.Substring (rootDirLength);
+					WriteFileToZip (log, archive, resource, zipName, maxCompression);
+				} else {
+					throw new FileNotFoundException (resource);
+				}
+				log.LogMessage (MessageImportance.Low, "Updated {0} with {1}", zip, resource);
+			}
+
+			return rv;
+		}
+
+		static void WriteFileToZip (TaskLoggingHelper log, ZipArchive archive, string path, string zipName, bool maxCompression)
+		{
+			var zipEntry = archive.CreateEntry (zipName, maxCompression ? SmallestCompressionLevel : CompressionLevel.Optimal);
+			using var fs = File.OpenRead (path);
+			using var zipStream = zipEntry.Open ();
+			fs.CopyTo (zipStream);
+			log.LogMessage (MessageImportance.Low, $"Compressed {path} into the zip file as {zipName}");
+		}
+
 		static int GetExternalAttributes (ZipArchiveEntry self)
 		{
 			// The ZipArchiveEntry.ExternalAttributes property is available in .NET 4.7.2 (which we need to target for builds on Windows) and .NET 5+, but not netstandard2.0 (which is the latest netstandard .NET 4.7.2 supports).

msbuild/Xamarin.MacDev.Tasks/Tasks/ComputeRemoteGeneratorProperties.cs

@@ -0,0 +1,40 @@
+using System.Linq;
+using System.Collections.Generic;
+
+using Xamarin.Messaging.Build.Client;
+using Microsoft.Build.Framework;
+
+#nullable enable
+
+namespace Xamarin.MacDev.Tasks {
+	public class ComputeRemoteGeneratorProperties : ComputeRemoteGeneratorPropertiesTaskBase, ITaskCallback, ICancelableTask {
+		public override bool Execute ()
+		{
+			bool result;
+
+			if (ShouldExecuteRemotely ()) {
+				result = new TaskRunner (SessionId, BuildEngine4).RunAsync (this).Result;
+			} else {
+				result = base.Execute ();
+			}
+
+			return result;
+		}
+
+		public void Cancel ()
+		{
+			if (ShouldExecuteRemotely ())
+				BuildConnection.CancelAsync (BuildEngine4).Wait ();
+		}
+
+		public bool ShouldCopyToBuildServer (ITaskItem item)
+		{
+			return false;
+		}
+
+		public bool ShouldCreateOutputFile (ITaskItem item) => false;
+
+		public IEnumerable<ITaskItem> GetAdditionalItemsToBeCopied () => Enumerable.Empty<ITaskItem> ();
+	}
+}
+

msbuild/Xamarin.MacDev.Tasks/Tasks/ComputeRemoteGeneratorPropertiesTaskBase.cs

@@ -2,19 +2,17 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Tasks;
 
 using Xamarin.Localization.MSBuild;
-using Xamarin.Messaging.Build.Client;
 using Xamarin.Utils;
 
 #nullable enable
 
 namespace Xamarin.MacDev.Tasks {
-	public class ComputeRemoteGeneratorProperties : XamarinTask, ITaskCallback, ICancelableTask {
+	public abstract class ComputeRemoteGeneratorPropertiesTaskBase : XamarinTask {
 		// Inputs
 		[Required]
 		public string IntermediateOutputPath { get; set; } = string.Empty;
@@ -39,9 +37,6 @@ public class ComputeRemoteGeneratorProperties : XamarinTask, ITaskCallback, ICan
 
 		public override bool Execute ()
 		{
-			if (ShouldExecuteRemotely ())
-				return new TaskRunner (SessionId, BuildEngine4).RunAsync (this).Result;
-
 			ComputeProperties ();
 
 			return !Log.HasLoggedErrors;
@@ -151,20 +146,5 @@ void ComputeProperties ()
 				}
 			}
 		}
-
-		public void Cancel ()
-		{
-			if (ShouldExecuteRemotely ())
-				BuildConnection.CancelAsync (BuildEngine4).Wait ();
-		}
-
-		public bool ShouldCopyToBuildServer (ITaskItem item)
-		{
-			return false;
-		}
-
-		public bool ShouldCreateOutputFile (ITaskItem item) => false;
-
-		public IEnumerable<ITaskItem> GetAdditionalItemsToBeCopied () => Enumerable.Empty<ITaskItem> ();
 	}
 }

msbuild/Xamarin.MacDev.Tasks/Tasks/CreateBindingResourcePackageBase.cs

@@ -77,19 +77,10 @@ public override bool Execute ()
 				filesToZip.Add (manifestPath);
 
 				foreach (var nativeRef in filesToZip) {
-					var zipArguments = new List<string> ();
-					zipArguments.Add ("-9");
-					zipArguments.Add ("-r");
-					zipArguments.Add ("-y");
-					zipArguments.Add (zipFile);
-
-					var fullPath = Path.GetFullPath (nativeRef);
-					var workingDirectory = Path.GetDirectoryName (fullPath);
-					zipArguments.Add (Path.GetFileName (fullPath));
-					ExecuteAsync ("zip", zipArguments, workingDirectory: workingDirectory).Wait ();
-
-					packagedFiles.Add (zipFile);
+					var workingDirectory = Path.GetDirectoryName (nativeRef);
+					CompressionHelper.TryCompress (Log, zipFile, new string [] { nativeRef }, false, workingDirectory, true);
 				}
+				packagedFiles.Add (zipFile);
 			} else {
 				var bindingResourcePath = BindingResourcePath;
 				Log.LogMessage (MSBStrings.M0121, bindingResourcePath);
@@ -114,11 +105,14 @@ public override bool Execute ()
 			return !Log.HasLoggedErrors;
 		}
 
-		static bool ContainsSymlinks (ITaskItem [] items)
+		bool ContainsSymlinks (ITaskItem [] items)
 		{
 			foreach (var item in items) {
-				if (PathUtils.IsSymlinkOrContainsSymlinks (item.ItemSpec))
+				if (PathUtils.IsSymlinkOrContainsSymlinks (item.ItemSpec)) {
+					if (Environment.OSVersion.Platform == PlatformID.Win32NT)
+						Log.LogError (MSBStrings.E7120 /* Can't process the native reference '{0}' on this platform because it is or contains a symlink. */, item.ItemSpec);
 					return true;
+				}
 			}
 
 			return false;

msbuild/Xamarin.MacDev.Tasks/Tasks/ZipTaskBase.cs

@@ -13,71 +13,21 @@
 #nullable disable
 
 namespace Xamarin.MacDev.Tasks {
-	public class Zip : XamarinToolTask, ITaskCallback {
+	public class Zip : XamarinTask, ITaskCallback {
 		#region Inputs
 
 		[Output]
 		[Required]
 		public ITaskItem OutputFile { get; set; }
 
-		public bool Recursive { get; set; }
-
 		[Required]
 		public ITaskItem [] Sources { get; set; }
 
-		public bool Symlinks { get; set; }
-
 		[Required]
 		public ITaskItem WorkingDirectory { get; set; }
 
 		#endregion
 
-		protected override string ToolName {
-			get { return "zip"; }
-		}
-
-		protected override string GenerateFullPathToTool ()
-		{
-			if (!string.IsNullOrEmpty (ToolPath))
-				return Path.Combine (ToolPath, ToolExe);
-
-			var path = Path.Combine ("/usr/bin", ToolExe);
-
-			return File.Exists (path) ? path : ToolExe;
-		}
-
-		protected override string GetWorkingDirectory ()
-		{
-			return WorkingDirectory.GetMetadata ("FullPath");
-		}
-
-		protected override string GenerateCommandLineCommands ()
-		{
-			var args = new CommandLineArgumentBuilder ();
-
-			if (Recursive)
-				args.Add ("-r");
-
-			if (Symlinks)
-				args.Add ("-y");
-
-			args.AddQuoted (OutputFile.GetMetadata ("FullPath"));
-
-			var root = WorkingDirectory.GetMetadata ("FullPath");
-			for (int i = 0; i < Sources.Length; i++) {
-				var relative = PathUtils.AbsoluteToRelative (root, Sources [i].GetMetadata ("FullPath"));
-				args.AddQuoted (relative);
-			}
-
-			return args.ToString ();
-		}
-
-		protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
-		{
-			// TODO: do proper parsing of error messages and such
-			Log.LogMessage (messageImportance, "{0}", singleLine);
-		}
-
 		public override bool Execute ()
 		{
 			if (ShouldExecuteRemotely ()) {
@@ -91,15 +41,24 @@ public override bool Execute ()
 				return rv;
 			}
 
-			return base.Execute ();
+			var zip = OutputFile.GetMetadata ("FullPath");
+			var workingDirectory = WorkingDirectory.GetMetadata ("FullPath");
+			var sources = new List<string> ();
+			for (int i = 0; i < Sources.Length; i++) {
+				var relative = PathUtils.AbsoluteToRelative (workingDirectory, Sources [i].GetMetadata ("FullPath"));
+				sources.Add (relative);
+			}
+
+			if (!CompressionHelper.TryCompress (this.Log, zip, sources, false, workingDirectory, false))
+				return false;
+
+			return !Log.HasLoggedErrors;
 		}
 
-		public override void Cancel ()
+		public void Cancel ()
 		{
 			if (ShouldExecuteRemotely ())
 				BuildConnection.CancelAsync (BuildEngine4).Wait ();
-
-			base.Cancel ();
 		}
 
 		public bool ShouldCopyToBuildServer (ITaskItem item) => false;