diff --git a/Build.proj b/Build.proj index 6352e423156b..6dcc3b954203 100644 --- a/Build.proj +++ b/Build.proj @@ -4,12 +4,11 @@ BuildTargetFramework=$([MSBuild]::ValueOrDefault('$(BuildTargetFramework)', '$(NetCoreAppCurrent)')) - + + + @@ -22,8 +21,8 @@ + BeforeTargets="Build"> + Targets="BuildIncrementally" /> \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props index 62a5c97bb10d..552f488feb43 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -70,7 +70,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) - $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'netstandard2.0', 'installer.tasks.dll')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'net461', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(AppleAppBuilderDir)', 'AppleAppBuilder.dll')) $([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll')) diff --git a/src/installer/publish/publish-final.proj b/src/installer/publish/publish-final.proj deleted file mode 100644 index d8eef2c55baa..000000000000 --- a/src/installer/publish/publish-final.proj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - - - - - - - - - - - - - - - InitializeSourceControlInformationFromSourceControlManager - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureBlobLease.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureBlobLease.cs deleted file mode 100644 index 7a84c407fc7e..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureBlobLease.cs +++ /dev/null @@ -1,182 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public partial class AzureBlobLease - { - private string _containerName; - private string _blobName; - private TimeSpan _maxWait; - private TimeSpan _delay; - private const int s_MaxWaitDefault = 60; // seconds - private const int s_DelayDefault = 500; // milliseconds - private CancellationTokenSource _cancellationTokenSource; - private Task _leaseRenewalTask; - private string _connectionString; - private string _accountName; - private string _accountKey; - private Microsoft.Build.Utilities.TaskLoggingHelper _log; - private string _leaseId; - private string _leaseUrl; - - public AzureBlobLease(string accountName, string accountKey, string connectionString, string containerName, string blobName, Microsoft.Build.Utilities.TaskLoggingHelper log, string maxWait = null, string delay = null) - { - _accountName = accountName; - _accountKey = accountKey; - _connectionString = connectionString; - _containerName = containerName; - _blobName = blobName; - _maxWait = !string.IsNullOrWhiteSpace(maxWait) ? TimeSpan.Parse(maxWait) : TimeSpan.FromSeconds(s_MaxWaitDefault); - _delay = !string.IsNullOrWhiteSpace(delay) ? TimeSpan.Parse(delay) : TimeSpan.FromMilliseconds(s_DelayDefault); - _log = log; - _leaseUrl = $"{AzureHelper.GetBlobRestUrl(_accountName, _containerName, _blobName)}?comp=lease"; - } - - public string Acquire() - { - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); - - while (stopWatch.ElapsedMilliseconds < _maxWait.TotalMilliseconds) - { - try - { - string leaseId = AcquireLeaseOnBlobAsync().GetAwaiter().GetResult(); - _cancellationTokenSource = new CancellationTokenSource(); - _leaseRenewalTask = Task.Run(() => - { AutoRenewLeaseOnBlob(this, _log); }, - _cancellationTokenSource.Token); - _leaseId = leaseId; - return _leaseId; - } - catch (Exception e) - { - _log.LogMessage($"Retrying lease acquisition on {_blobName}, {e.Message}"); - Thread.Sleep(_delay); - } - } - ResetLeaseRenewalTaskState(); - throw new Exception($"Unable to acquire lease on {_blobName}"); - - } - - public void Release() - { - // Cancel the lease renewal task since we are about to release the lease. - ResetLeaseRenewalTaskState(); - - using (HttpClient client = new HttpClient()) - { - Tuple leaseAction = new Tuple("x-ms-lease-action", "release"); - Tuple headerLeaseId = new Tuple("x-ms-lease-id", _leaseId); - List> additionalHeaders = new List>() { leaseAction, headerLeaseId }; - var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders); - using (HttpResponseMessage response = AzureHelper.RequestWithRetry(_log, client, request).GetAwaiter().GetResult()) - { - if (!response.IsSuccessStatusCode) - { - _log.LogMessage($"Unable to release lease on container/blob {_containerName}/{_blobName}."); - } - } - } - } - - private async Task AcquireLeaseOnBlobAsync() - { - _log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{_containerName}/{_blobName}'."); - string leaseId = string.Empty; - using (HttpClient client = new HttpClient()) - { - try - { - Tuple leaseAction = new Tuple("x-ms-lease-action", "acquire"); - Tuple leaseDuration = new Tuple("x-ms-lease-duration", "60" /* seconds */); - List> additionalHeaders = new List>() { leaseAction, leaseDuration }; - var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders); - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(_log, client, request)) - { - leaseId = response.Headers.GetValues("x-ms-lease-id").FirstOrDefault(); - } - } - catch (Exception e) - { - _log.LogErrorFromException(e, true); - } - } - - return leaseId; - } - private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Build.Utilities.TaskLoggingHelper log) - { - TimeSpan maxWait = TimeSpan.FromSeconds(s_MaxWaitDefault); - TimeSpan delay = TimeSpan.FromMilliseconds(s_DelayDefault); - TimeSpan waitFor = maxWait; - CancellationToken token = instance._cancellationTokenSource.Token; - - while (true) - { - token.ThrowIfCancellationRequested(); - - try - { - log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{instance._containerName}/{instance._blobName}'."); - using (HttpClient client = new HttpClient()) - { - Tuple leaseAction = new Tuple("x-ms-lease-action", "renew"); - Tuple headerLeaseId = new Tuple("x-ms-lease-id", instance._leaseId); - List> additionalHeaders = new List>() { leaseAction, headerLeaseId }; - var request = AzureHelper.RequestMessage("PUT", instance._leaseUrl, instance._accountName, instance._accountKey, additionalHeaders); - using (HttpResponseMessage response = AzureHelper.RequestWithRetry(log, client, request).GetAwaiter().GetResult()) - { - if (!response.IsSuccessStatusCode) - { - throw new Exception("Unable to acquire lease."); - } - } - } - waitFor = maxWait; - } - catch (Exception e) - { - Console.WriteLine($"Rerying lease renewal on {instance._containerName}, {e.Message}"); - waitFor = delay; - } - token.ThrowIfCancellationRequested(); - - Task.Delay(waitFor, token).Wait(); - } - } - - private void ResetLeaseRenewalTaskState() - { - // Cancel the lease renewal task if it was created - if (_leaseRenewalTask != null) - { - _cancellationTokenSource.Cancel(); - - // Block until the task ends. It can throw if we cancelled it before it completed. - try - { - _leaseRenewalTask.Wait(); - } - catch (Exception) - { - // Ignore the caught exception as it will be expected. - } - - _leaseRenewalTask = null; - } - } - - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureConnectionStringBuildTask.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureConnectionStringBuildTask.cs deleted file mode 100644 index 82faa5ab5146..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureConnectionStringBuildTask.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Utilities; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public abstract class AzureConnectionStringBuildTask : Task - { - /// - /// Azure Storage account connection string. Supersedes Account Key / Name. - /// Will cause errors if both are set. - /// - public string ConnectionString { get; set; } - - /// - /// The Azure account key used when creating the connection string. - /// When we fully deprecate these, can just make them get; only. - /// - public string AccountKey { get; set; } - - /// - /// The Azure account name used when creating the connection string. - /// When we fully deprecate these, can just make them get; only. - /// - public string AccountName { get; set; } - - public void ParseConnectionString() - { - if (!string.IsNullOrEmpty(ConnectionString)) - { - if (!(string.IsNullOrEmpty(AccountKey) && string.IsNullOrEmpty(AccountName))) - { - Log.LogError("If the ConnectionString property is set, you must not provide AccountKey / AccountName. These values will be deprecated in the future."); - } - else - { - Regex storageConnectionStringRegex = new Regex("AccountName=(?.+?);AccountKey=(?.+?);"); - - MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString); - if (matches.Count > 0) - { - // When we deprecate this format, we'll want to demote these to private - AccountName = matches[0].Groups["name"].Value; - AccountKey = matches[0].Groups["key"].Value; - } - else - { - Log.LogError("Error parsing connection string. Please review its value."); - } - } - } - else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName)) - { - Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey"); - } - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureHelper.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureHelper.cs deleted file mode 100644 index 5e35ad6ddfe0..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureHelper.cs +++ /dev/null @@ -1,460 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public static class AzureHelper - { - /// - /// The storage api version. - /// - public static readonly string StorageApiVersion = "2015-04-05"; - public const string DateHeaderString = "x-ms-date"; - public const string VersionHeaderString = "x-ms-version"; - public const string AuthorizationHeaderString = "Authorization"; - public const string CacheControlString = "x-ms-blob-cache-control"; - public const string ContentTypeString = "x-ms-blob-content-type"; - - public enum SasAccessType - { - Read, - Write, - }; - - public static string AuthorizationHeader( - string storageAccount, - string storageKey, - string method, - DateTime now, - HttpRequestMessage request, - string ifMatch = "", - string contentMD5 = "", - string size = "", - string contentType = "") - { - string stringToSign = string.Format( - "{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}", - method, - (size == string.Empty) ? string.Empty : size, - ifMatch, - GetCanonicalizedHeaders(request), - GetCanonicalizedResource(request.RequestUri, storageAccount), - contentMD5, - contentType); - byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); - string authorizationHeader; - using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey))) - { - authorizationHeader = "SharedKey " + storageAccount + ":" - + Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes)); - } - - return authorizationHeader; - } - - public static string CreateContainerSasToken( - string accountName, - string containerName, - string key, - SasAccessType accessType, - int validityTimeInDays) - { - string signedPermissions = string.Empty; - switch (accessType) - { - case SasAccessType.Read: - signedPermissions = "r"; - break; - case SasAccessType.Write: - signedPermissions = "wdl"; - break; - default: - throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value"); - } - - string signedStart = DateTime.UtcNow.ToString("O"); - string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O"); - string canonicalizedResource = "/blob/" + accountName + "/" + containerName; - string signedIdentifier = string.Empty; - string signedVersion = StorageApiVersion; - - string stringToSign = ConstructServiceStringToSign( - signedPermissions, - signedVersion, - signedExpiry, - canonicalizedResource, - signedIdentifier, - signedStart); - - byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); - string signature; - using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key))) - { - signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes)); - } - - string sasToken = string.Format( - "?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}", - WebUtility.UrlEncode(signedVersion), - WebUtility.UrlEncode("c"), - WebUtility.UrlEncode(signature), - WebUtility.UrlEncode(signedStart), - WebUtility.UrlEncode(signedExpiry), - WebUtility.UrlEncode(signedPermissions)); - - return sasToken; - } - - public static string GetCanonicalizedHeaders(HttpRequestMessage request) - { - StringBuilder sb = new StringBuilder(); - List headerNameList = (from headerName in request.Headers - where - headerName.Key.ToLowerInvariant() - .StartsWith("x-ms-", StringComparison.Ordinal) - select headerName.Key.ToLowerInvariant()).ToList(); - headerNameList.Sort(); - foreach (string headerName in headerNameList) - { - StringBuilder builder = new StringBuilder(headerName); - string separator = ":"; - foreach (string headerValue in GetHeaderValues(request.Headers, headerName)) - { - string trimmedValue = headerValue.Replace("\r\n", string.Empty); - builder.Append(separator); - builder.Append(trimmedValue); - separator = ","; - } - - sb.Append(builder); - sb.Append("\n"); - } - - return sb.ToString(); - } - - public static string GetCanonicalizedResource(Uri address, string accountName) - { - StringBuilder str = new StringBuilder(); - StringBuilder builder = new StringBuilder("/"); - builder.Append(accountName); - builder.Append(address.AbsolutePath); - str.Append(builder); - Dictionary> queryKeyValues = ExtractQueryKeyValues(address); - Dictionary> dictionary = GetCommaSeparatedList(queryKeyValues); - - foreach (KeyValuePair> pair in dictionary.OrderBy(p => p.Key)) - { - StringBuilder stringBuilder = new StringBuilder(string.Empty); - stringBuilder.Append(pair.Key + ":"); - string commaList = string.Join(",", pair.Value); - stringBuilder.Append(commaList); - str.Append("\n"); - str.Append(stringBuilder); - } - - return str.ToString(); - } - - public static List GetHeaderValues(HttpRequestHeaders headers, string headerName) - { - List list = new List(); - IEnumerable values; - headers.TryGetValues(headerName, out values); - if (values != null) - { - list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null)); - } - - return list; - } - - private static bool IsWithinRetryRange(HttpStatusCode statusCode) - { - // Retry on http client and server error codes (4xx - 5xx) as well as redirect - - var rawStatus = (int)statusCode; - if (rawStatus == 302) - return true; - else if (rawStatus >= 400 && rawStatus <= 599) - return true; - else - return false; - } - - public static async Task RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client, - Func createRequest, Func validationCallback = null, int retryCount = 5, - int retryDelaySeconds = 5) - { - if (loggingHelper == null) - throw new ArgumentNullException(nameof(loggingHelper)); - if (client == null) - throw new ArgumentNullException(nameof(client)); - if (createRequest == null) - throw new ArgumentNullException(nameof(createRequest)); - if (retryCount < 1) - throw new ArgumentException(nameof(retryCount)); - if (retryDelaySeconds < 1) - throw new ArgumentException(nameof(retryDelaySeconds)); - - int retries = 0; - HttpResponseMessage response = null; - - // add a bit of randomness to the retry delay - var rng = new Random(); - - while (retries < retryCount) - { - if (retries > 0) - { - if (response != null) - { - response.Dispose(); - response = null; - } - - int delay = retryDelaySeconds * retries * rng.Next(1, 5); - loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay); - await System.Threading.Tasks.Task.Delay(delay * 1000); - } - - try - { - using (var request = createRequest()) - response = await client.SendAsync(request); - } - catch (Exception e) - { - loggingHelper.LogWarningFromException(e, true); - - // if this is the final iteration let the exception bubble up - if (retries + 1 == retryCount) - throw; - } - - // response can be null if we fail to send the request - if (response != null) - { - if (validationCallback == null) - { - // check if the response code is within the range of failures - if (!IsWithinRetryRange(response.StatusCode)) - { - return response; - } - } - else - { - bool isSuccess = validationCallback(response); - if (!isSuccess) - { - loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode); - } - else - { - loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode); - return response; - } - } - } - - ++retries; - } - - // retry count exceeded - loggingHelper.LogWarning("Retry count {0} exceeded", retryCount); - - // set some default values in case response is null - var statusCode = "None"; - var contentStr = "Null"; - if (response != null) - { - statusCode = response.StatusCode.ToString(); - contentStr = await response.Content.ReadAsStringAsync(); - response.Dispose(); - } - - throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}"); - } - - private static string ConstructServiceStringToSign( - string signedPermissions, - string signedVersion, - string signedExpiry, - string canonicalizedResource, - string signedIdentifier, - string signedStart, - string signedIP = "", - string signedProtocol = "", - string rscc = "", - string rscd = "", - string rsce = "", - string rscl = "", - string rsct = "") - { - // constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx - var stringToSign = string.Join( - "\n", - signedPermissions, - signedStart, - signedExpiry, - canonicalizedResource, - signedIdentifier, - signedIP, - signedProtocol, - signedVersion, - rscc, - rscd, - rsce, - rscl, - rsct); - return stringToSign; - } - - private static Dictionary> ExtractQueryKeyValues(Uri address) - { - Dictionary> values = new Dictionary>(); - //Decode this to allow the regex to pull out the correct groups for signing - address = new Uri(WebUtility.UrlDecode(address.ToString())); - Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)"); - MatchCollection matches = newreg.Matches(address.Query); - foreach (Match match in matches) - { - string key, value; - if (!string.IsNullOrEmpty(match.Groups[1].Value)) - { - key = match.Groups[1].Value; - value = match.Groups[2].Value; - } - else - { - key = match.Groups[3].Value; - value = match.Groups[4].Value; - } - - HashSet setOfValues; - if (values.TryGetValue(key, out setOfValues)) - { - setOfValues.Add(value); - } - else - { - HashSet newSet = new HashSet { value }; - values.Add(key, newSet); - } - } - - return values; - } - - private static Dictionary> GetCommaSeparatedList( - Dictionary> queryKeyValues) - { - Dictionary> dictionary = new Dictionary>(); - - foreach (string queryKeys in queryKeyValues.Keys) - { - HashSet setOfValues; - queryKeyValues.TryGetValue(queryKeys, out setOfValues); - List list = new List(); - list.AddRange(setOfValues); - list.Sort(); - string commaSeparatedValues = string.Join(",", list); - string key = queryKeys.ToLowerInvariant(); - HashSet setOfValues2; - if (dictionary.TryGetValue(key, out setOfValues2)) - { - setOfValues2.Add(commaSeparatedValues); - } - else - { - HashSet newSet = new HashSet { commaSeparatedValues }; - dictionary.Add(key, newSet); - } - } - - return dictionary; - } - - public static Func RequestMessage(string method, string url, string accountName, string accountKey, List> additionalHeaders = null, string body = null) - { - Func requestFunc = () => - { - HttpMethod httpMethod = HttpMethod.Get; - if (method == "PUT") - { - httpMethod = HttpMethod.Put; - } - else if (method == "DELETE") - { - httpMethod = HttpMethod.Delete; - } - DateTime dateTime = DateTime.UtcNow; - var request = new HttpRequestMessage(httpMethod, url); - request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); - request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (additionalHeaders != null) - { - foreach (Tuple additionalHeader in additionalHeaders) - { - request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2); - } - } - if (body != null) - { - request.Content = new StringContent(body); - request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( - accountName, - accountKey, - method, - dateTime, - request, - "", - "", - request.Content.Headers.ContentLength.ToString(), - request.Content.Headers.ContentType.ToString())); - } - else - { - request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( - accountName, - accountKey, - method, - dateTime, - request)); - } - return request; - }; - return requestFunc; - } - - public static string GetRootRestUrl(string accountName) - { - return $"https://{accountName}.blob.core.windows.net"; - } - - public static string GetContainerRestUrl(string accountName, string containerName) - { - return $"{GetRootRestUrl(accountName)}/{containerName}"; - } - - public static string GetBlobRestUrl(string accountName, string containerName, string blob) - { - return $"{GetContainerRestUrl(accountName, containerName)}/{blob}"; - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/CopyAzureBlobToBlob.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/CopyAzureBlobToBlob.cs deleted file mode 100644 index 8001de8e11d0..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/CopyAzureBlobToBlob.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public partial class CopyAzureBlobToBlob : AzureConnectionStringBuildTask - { - [Required] - public string ContainerName { get; set; } - [Required] - public string SourceBlobName { get; set; } - [Required] - public string DestinationBlobName { get; set; } - - public override bool Execute() - { - return ExecuteAsync().GetAwaiter().GetResult(); - } - - public async Task ExecuteAsync() - { - ParseConnectionString(); - if (Log.HasLoggedErrors) - { - return false; - } - - string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName); - string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName); - using (HttpClient client = new HttpClient()) - { - try - { - Tuple leaseAction = new Tuple("x-ms-lease-action", "acquire"); - Tuple leaseDuration = new Tuple("x-ms-lease-duration", "60" /* seconds */); - Tuple headerSource = new Tuple("x-ms-copy-source", sourceUrl); - List> additionalHeaders = new List>() { leaseAction, leaseDuration, headerSource }; - var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders); - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request)) - { - if (response.IsSuccessStatusCode) - { - return true; - } - } - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - } - return false; - } - public static bool Execute(string accountName, - string accountKey, - string connectionString, - string containerName, - string sourceBlobName, - string destinationBlobName, - IBuildEngine buildengine, - ITaskHost taskHost) - { - CopyAzureBlobToBlob copyAzureBlobToBlob = new CopyAzureBlobToBlob() - { - AccountName = accountName, - AccountKey = accountKey, - ContainerName = containerName, - SourceBlobName = sourceBlobName, - DestinationBlobName = destinationBlobName, - BuildEngine = buildengine, - HostObject = taskHost - }; - return copyAzureBlobToBlob.Execute(); - } - public static Task ExecuteAsync(string accountName, - string accountKey, - string connectionString, - string containerName, - string sourceBlobName, - string destinationBlobName, - IBuildEngine buildengine, - ITaskHost taskHost) - { - CopyAzureBlobToBlob copyAzureBlobToBlob = new CopyAzureBlobToBlob() - { - AccountName = accountName, - AccountKey = accountKey, - ContainerName = containerName, - SourceBlobName = sourceBlobName, - DestinationBlobName = destinationBlobName, - BuildEngine = buildengine, - HostObject = taskHost - }; - return copyAzureBlobToBlob.ExecuteAsync(); - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DeleteAzureBlob.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DeleteAzureBlob.cs deleted file mode 100644 index 92682226012c..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DeleteAzureBlob.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.Net.Http; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public partial class DeleteAzureBlob: AzureConnectionStringBuildTask - { - [Required] - public string ContainerName { get; set; } - [Required] - public string BlobName { get; set; } - - public override bool Execute() - { - ParseConnectionString(); - if (Log.HasLoggedErrors) - { - return false; - } - - string deleteUrl = $"https://{AccountName}.blob.core.windows.net/{ContainerName}/{BlobName}"; - - using (HttpClient client = new HttpClient()) - { - try - { - Tuple snapshots = new Tuple("x-ms-lease-delete-snapshots", "include"); - List> additionalHeaders = new List>() { snapshots }; - var request = AzureHelper.RequestMessage("DELETE", deleteUrl, AccountName, AccountKey, additionalHeaders); - using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult()) - { - return response.IsSuccessStatusCode; - } - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - } - - return !Log.HasLoggedErrors; - } - - public static bool Execute(string accountName, - string accountKey, - string connectionString, - string containerName, - string blobName, - IBuildEngine buildengine, - ITaskHost taskHost) - { - DeleteAzureBlob deleteAzureoBlob = new DeleteAzureBlob() - { - AccountName = accountName, - AccountKey = accountKey, - ContainerName = containerName, - BlobName = blobName, - BuildEngine = buildengine, - HostObject = taskHost - }; - return deleteAzureoBlob.Execute(); - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DownloadFromAzure.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DownloadFromAzure.cs deleted file mode 100644 index 6fb1d671ee09..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DownloadFromAzure.cs +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public sealed class DownloadFromAzure : AzureConnectionStringBuildTask - { - /// - /// The name of the container to access. The specified name must be in the correct format, see the - /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx - /// - [Required] - public string ContainerName { get; set; } - - /// - /// Directory to download blob files to. - /// - [Required] - public string DownloadDirectory { get; set; } - - public string BlobNamePrefix { get; set; } - - public string BlobNameExtension { get; set; } - - public ITaskItem[] BlobNames { get; set; } - - public bool DownloadFlatFiles { get; set; } - - public int MaxClients { get; set; } = 8; - - private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource(); - private static readonly CancellationToken CancellationToken = TokenSource.Token; - - public void Cancel() - { - TokenSource.Cancel(); - } - - public override bool Execute() - { - return ExecuteAsync(CancellationToken).GetAwaiter().GetResult(); - } - - public async Task ExecuteAsync(CancellationToken ct) - { - ParseConnectionString(); - // If the connection string AND AccountKey & AccountName are provided, error out. - if (Log.HasLoggedErrors) - { - return false; - } - - Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.", - ContainerName, AccountName, DownloadDirectory); - - try - { - List blobNames = new List(); - if (BlobNames == null) - { - ListAzureBlobs listAzureBlobs = new ListAzureBlobs() - { - AccountName = AccountName, - AccountKey = AccountKey, - ContainerName = ContainerName, - FilterBlobNames = BlobNamePrefix, - BuildEngine = this.BuildEngine, - HostObject = this.HostObject - }; - listAzureBlobs.Execute(); - blobNames = listAzureBlobs.BlobNames.ToList(); - } - else - { - blobNames = BlobNames.Select(b => b.ItemSpec).ToList(); - if (BlobNamePrefix != null) - { - blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList(); - } - } - - if (BlobNameExtension != null) - { - blobNames = blobNames.Where(b => Path.GetExtension(b) == BlobNameExtension).ToList(); - } - - if (!Directory.Exists(DownloadDirectory)) - { - Directory.CreateDirectory(DownloadDirectory); - } - using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) - using (HttpClient client = new HttpClient()) - { - client.Timeout = TimeSpan.FromMinutes(10); - await Task.WhenAll(blobNames.Select(item => DownloadItem(client, ct, item, clientThrottle))); - } - } - catch (Exception e) - { - Log.LogError(e.ToString()); - } - return !Log.HasLoggedErrors; - } - - private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle) - { - await clientThrottle.WaitAsync(); - string filename = string.Empty; - try { - Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); - string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); - filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob)); - - if (!DownloadFlatFiles) - { - int dirIndex = blob.LastIndexOf("/"); - string blobDirectory = string.Empty; - string blobFilename = string.Empty; - - if (dirIndex == -1) - { - blobFilename = blob; - } - else - { - blobDirectory = blob.Substring(0, dirIndex); - blobFilename = blob.Substring(dirIndex + 1); - - // Trim blob name prefix (directory part) from download to blob directory - if (BlobNamePrefix != null) - { - if (BlobNamePrefix.Length > dirIndex) - { - BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex); - } - blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length); - } - } - string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); - if (!Directory.Exists(downloadBlobDirectory)) - { - Directory.CreateDirectory(downloadBlobDirectory); - } - filename = Path.Combine(downloadBlobDirectory, blobFilename); - } - - var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey); - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) - { - if (response.IsSuccessStatusCode) - { - // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. - if (!string.IsNullOrEmpty(Path.GetFileName(filename))) - { - Stream responseStream = await response.Content.ReadAsStreamAsync(); - - using (FileStream sourceStream = File.Open(filename, FileMode.Create)) - { - responseStream.CopyTo(sourceStream); - } - } - else - { - Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); - } - } - else - { - Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); - } - } - } - catch (PathTooLongException) - { - Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}"); - } - catch (Exception ex) - { - Log.LogError(ex.ToString()); - } - finally - { - clientThrottle.Release(); - } - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/ListAzureBlobs.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/ListAzureBlobs.cs deleted file mode 100644 index 2376d5638597..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/ListAzureBlobs.cs +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using System.Xml; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public partial class ListAzureBlobs : AzureConnectionStringBuildTask - { - - /// - /// The name of the container to access. The specified name must be in the correct format, see the - /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx - /// - [Required] - public string ContainerName { get; set; } - - [Output] - public string[] BlobNames { get; set; } - - public string FilterBlobNames { get; set; } - - public override bool Execute() - { - return ExecuteAsync().GetAwaiter().GetResult(); - } - - public static string[] Execute(string accountName, - string accountKey, - string connectionString, - string containerName, - string filterBlobNames, - IBuildEngine buildengine, - ITaskHost taskHost) - { - ListAzureBlobs getAzureBlobList = new ListAzureBlobs() - { - AccountName = accountName, - AccountKey = accountKey, - ContainerName = containerName, - FilterBlobNames = filterBlobNames, - BuildEngine = buildengine, - HostObject = taskHost - }; - getAzureBlobList.Execute(); - return getAzureBlobList.BlobNames; - } - - // This code is duplicated in BuildTools task DownloadFromAzure, and that code should be refactored to permit blob listing. - public async Task ExecuteAsync() - { - ParseConnectionString(); - try - { - List blobNames = await ListBlobs(Log, AccountName, AccountKey, ContainerName, FilterBlobNames); - BlobNames = blobNames.ToArray(); - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - return !Log.HasLoggedErrors; - } - - public static async Task> ListBlobs(TaskLoggingHelper Log, string AccountName, string AccountKey, string ContainerName, string FilterBlobNames) - { - List blobsNames = new List(); - string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); - if (!string.IsNullOrWhiteSpace(FilterBlobNames)) - { - urlListBlobs += $"&prefix={FilterBlobNames}"; - } - Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName); - - using (HttpClient client = new HttpClient()) - { - - var createRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey); - - XmlDocument responseFile; - string nextMarker = string.Empty; - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) - { - responseFile = new XmlDocument(); - responseFile.LoadXml(await response.Content.ReadAsStringAsync()); - XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); - - blobsNames.AddRange(elemList.Cast() - .Select(x => x.InnerText) - .ToList()); - - nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast().FirstOrDefault()?.InnerText; - } - while (!string.IsNullOrEmpty(nextMarker)) - { - urlListBlobs = $"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}"; - if (!string.IsNullOrWhiteSpace(FilterBlobNames)) - { - urlListBlobs += $"&prefix={FilterBlobNames}"; - } - var nextRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey); - using (HttpResponseMessage nextResponse = AzureHelper.RequestWithRetry(Log, client, nextRequest).GetAwaiter().GetResult()) - { - responseFile = new XmlDocument(); - responseFile.LoadXml(await nextResponse.Content.ReadAsStringAsync()); - XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); - - blobsNames.AddRange(elemList.Cast() - .Select(x => x.InnerText) - .ToList()); - - nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast().FirstOrDefault()?.InnerText; - } - } - } - return blobsNames; - } - } - -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/PublishStringToAzureBlob.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/PublishStringToAzureBlob.cs deleted file mode 100644 index 79293817d630..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/PublishStringToAzureBlob.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.Net.Http; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public partial class PublishStringToAzureBlob : AzureConnectionStringBuildTask - { - [Required] - public string BlobName { get; set; } - [Required] - public string ContainerName { get; set; } - [Required] - public string Content { get; set; } - public string ContentType { get; set; } - - public override bool Execute() - { - ParseConnectionString(); - - string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, BlobName); - using (HttpClient client = new HttpClient()) - { - try - { - Tuple headerBlobType = new Tuple("x-ms-blob-type", "BlockBlob"); - List> additionalHeaders = new List>() { headerBlobType }; - - if (!string.IsNullOrEmpty(ContentType)) - { - additionalHeaders.Add(new Tuple(AzureHelper.ContentTypeString, ContentType)); - } - - var request = AzureHelper.RequestMessage("PUT", blobUrl, AccountName, AccountKey, additionalHeaders, Content); - - AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult(); - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - } - - return !Log.HasLoggedErrors; - } - - public static bool Execute(string accountName, - string accountKey, - string connectionString, - string containerName, - string blobName, - string content, - string contentType, - IBuildEngine buildengine, - ITaskHost taskHost) - { - PublishStringToAzureBlob publishStringToBlob = new PublishStringToAzureBlob() - { - AccountName = accountName, - AccountKey = accountKey, - ContainerName = containerName, - BlobName = blobName, - Content = content, - ContentType = contentType, - BuildEngine = buildengine, - HostObject = taskHost - }; - return publishStringToBlob.Execute(); - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadClient.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadClient.cs deleted file mode 100644 index fdbc307bfe8e..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadClient.cs +++ /dev/null @@ -1,284 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - public class UploadClient - { - private TaskLoggingHelper log; - - public UploadClient(TaskLoggingHelper loggingHelper) - { - log = loggingHelper; - } - - public string EncodeBlockIds(int numberOfBlocks, int lengthOfId) - { - string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId); - if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64) - { - byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString); - return Convert.ToBase64String(bytes); - } - else - { - throw new Exception("Task failed - Could not encode block id."); - } - } - - public async Task UploadBlockBlobAsync( - CancellationToken ct, - string AccountName, - string AccountKey, - string ContainerName, - string filePath, - string destinationBlob, - string contentType, - int uploadTimeout, - string leaseId = "") - { - string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName); - - string fileName = destinationBlob; - fileName = fileName.Replace("\\", "/"); - string blobUploadUrl = resourceUrl + "/" + fileName; - int size = (int)new FileInfo(filePath).Length; - int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob - int bytesLeft = size; - List blockIds = new List(); - int numberOfBlocks = (size / blockSize) + 1; - int countForId = 0; - using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read)) - { - int offset = 0; - - while (bytesLeft > 0) - { - int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; - byte[] fileBytes = new byte[blockSize]; - int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); - - if (nextBytesToRead != read) - { - throw new Exception(string.Format( - "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", - read, fileName, nextBytesToRead)); - } - - string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); - - blockIds.Add(blockId); - string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); - - using (HttpClient client = new HttpClient()) - { - client.DefaultRequestHeaders.Clear(); - - // In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB. - client.Timeout = TimeSpan.FromMinutes(uploadTimeout); - Func createRequest = () => - { - DateTime dt = DateTime.UtcNow; - var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); - req.Headers.Add( - AzureHelper.DateHeaderString, - dt.ToString("R", CultureInfo.InvariantCulture)); - req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (!string.IsNullOrWhiteSpace(leaseId)) - { - log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}"); - req.Headers.Add("x-ms-lease-id", leaseId); - } - req.Headers.Add( - AzureHelper.AuthorizationHeaderString, - AzureHelper.AuthorizationHeader( - AccountName, - AccountKey, - "PUT", - dt, - req, - string.Empty, - string.Empty, - nextBytesToRead.ToString(), - string.Empty)); - - Stream postStream = new MemoryStream(); - postStream.Write(fileBytes, 0, nextBytesToRead); - postStream.Seek(0, SeekOrigin.Begin); - req.Content = new StreamContent(postStream); - return req; - }; - - log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) - { - log.LogMessage( - MessageImportance.Low, - "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", - countForId, - fileName, - response.StatusCode, - await response.Content.ReadAsStringAsync()); - } - } - - offset += read; - bytesLeft -= nextBytesToRead; - countForId += 1; - } - } - - string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; - - using (HttpClient client = new HttpClient()) - { - Func createRequest = () => - { - DateTime dt1 = DateTime.UtcNow; - var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); - req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); - req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (string.IsNullOrEmpty(contentType)) - { - contentType = DetermineContentTypeBasedOnFileExtension(filePath); - } - if (!string.IsNullOrEmpty(contentType)) - { - req.Headers.Add(AzureHelper.ContentTypeString, contentType); - } - string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath); - if (!string.IsNullOrEmpty(cacheControl)) - { - req.Headers.Add(AzureHelper.CacheControlString, cacheControl); - } - - var body = new StringBuilder(""); - foreach (object item in blockIds) - body.AppendFormat("{0}", item); - - body.Append(""); - byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); - if (!string.IsNullOrWhiteSpace(leaseId)) - { - log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}"); - req.Headers.Add("x-ms-lease-id", leaseId); - } - req.Headers.Add( - AzureHelper.AuthorizationHeaderString, - AzureHelper.AuthorizationHeader( - AccountName, - AccountKey, - "PUT", - dt1, - req, - string.Empty, - string.Empty, - bodyData.Length.ToString(), - string.Empty)); - - Stream postStream = new MemoryStream(); - postStream.Write(bodyData, 0, bodyData.Length); - postStream.Seek(0, SeekOrigin.Begin); - req.Content = new StreamContent(postStream); - return req; - }; - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) - { - log.LogMessage( - MessageImportance.Low, - "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", - fileName, - response.StatusCode, - await response.Content.ReadAsStringAsync()); - } - } - } - - public async Task FileEqualsExistingBlobAsync( - string accountName, - string accountKey, - string containerName, - string filePath, - string destinationBlob, - int uploadTimeout) - { - using (var client = new HttpClient - { - Timeout = TimeSpan.FromMinutes(uploadTimeout) - }) - { - log.LogMessage( - MessageImportance.Low, - $"Downloading blob {destinationBlob} to check if identical."); - - string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); - var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( - log, - client, - createRequest)) - { - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException( - $"Failed to retrieve existing blob {destinationBlob}, " + - $"status code {response.StatusCode}."); - } - - byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); - byte[] localBytes = File.ReadAllBytes(filePath); - - bool equal = localBytes.SequenceEqual(existingBytes); - - if (equal) - { - log.LogMessage( - MessageImportance.Normal, - "Item exists in blob storage, and is verified to be identical. " + - $"File: '{filePath}' Blob: '{destinationBlob}'"); - } - - return equal; - } - } - } - - private string DetermineContentTypeBasedOnFileExtension(string filename) - { - if (Path.GetExtension(filename) == ".svg") - { - return "image/svg+xml"; - } - else if (Path.GetExtension(filename) == ".version") - { - return "text/plain"; - } - return string.Empty; - } - private string DetermineCacheControlBasedOnFileExtension(string filename) - { - if (Path.GetExtension(filename) == ".svg") - { - return "No-Cache"; - } - return string.Empty; - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadToAzure.cs b/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadToAzure.cs deleted file mode 100644 index 4befc1e5abf9..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadToAzure.cs +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using ThreadingTask = System.Threading.Tasks.Task; - -namespace Microsoft.DotNet.Build.CloudTestTasks -{ - - public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask - { - private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource(); - private static readonly CancellationToken CancellationToken = TokenSource.Token; - - /// - /// The name of the container to access. The specified name must be in the correct format, see the - /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx - /// - [Required] - public string ContainerName { get; set; } - - /// - /// An item group of files to upload. Each item must have metadata RelativeBlobPath - /// that specifies the path relative to ContainerName where the item will be uploaded. - /// - [Required] - public ITaskItem[] Items { get; set; } - - /// - /// Indicates if the destination blob should be overwritten if it already exists. The default if false. - /// - public bool Overwrite { get; set; } = false; - - /// - /// Enables idempotency when Overwrite is false. - /// - /// false: (default) Attempting to upload an item that already exists fails. - /// - /// true: When an item already exists, download the existing blob to check if it's - /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. - /// - public bool PassIfExistingItemIdentical { get; set; } - - /// - /// Specifies the maximum number of clients to concurrently upload blobs to azure - /// - public int MaxClients { get; set; } = 8; - - public int UploadTimeoutInMinutes { get; set; } = 5; - - public void Cancel() - { - TokenSource.Cancel(); - } - - public override bool Execute() - { - return ExecuteAsync(CancellationToken).GetAwaiter().GetResult(); - } - - public async Task ExecuteAsync(CancellationToken ct) - { - ParseConnectionString(); - // If the connection string AND AccountKey & AccountName are provided, error out. - if (Log.HasLoggedErrors) - { - return false; - } - - Log.LogMessage( - MessageImportance.Normal, - "Begin uploading blobs to Azure account {0} in container {1}.", - AccountName, - ContainerName); - - if (Items.Length == 0) - { - Log.LogError("No items were provided for upload."); - return false; - } - - // first check what blobs are present - string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; - - HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase); - - try - { - using (HttpClient client = new HttpClient()) - { - var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); - - Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) - { - var doc = new XmlDocument(); - doc.LoadXml(await response.Content.ReadAsStringAsync()); - - XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); - - foreach (XmlNode node in nodes) - { - blobsPresent.Add(node["Name"].InnerText); - } - - Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); - } - } - - using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) - { - await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); - } - - Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - return !Log.HasLoggedErrors; - } - - private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet blobsPresent, SemaphoreSlim clientThrottle) - { - if (ct.IsCancellationRequested) - { - Log.LogError("Task UploadToAzure cancelled"); - ct.ThrowIfCancellationRequested(); - } - - string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); - if (string.IsNullOrEmpty(relativeBlobPath)) - throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec)); - - if (!File.Exists(item.ItemSpec)) - throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); - - UploadClient uploadClient = new UploadClient(Log); - - if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) - { - if (PassIfExistingItemIdentical && - await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) - { - return; - } - - throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); - } - - string contentType = item.GetMetadata("ContentType"); - - await clientThrottle.WaitAsync(); - - try - { - Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); - await - uploadClient.UploadBlockBlobAsync( - ct, - AccountName, - AccountKey, - ContainerName, - item.ItemSpec, - relativeBlobPath, - contentType, - UploadTimeoutInMinutes); - } - finally - { - clientThrottle.Release(); - } - } - - private async Task ItemEqualsExistingBlobAsync( - ITaskItem item, - string relativeBlobPath, - UploadClient client, - SemaphoreSlim clientThrottle) - { - await clientThrottle.WaitAsync(); - try - { - return await client.FileEqualsExistingBlobAsync( - AccountName, - AccountKey, - ContainerName, - item.ItemSpec, - relativeBlobPath, - UploadTimeoutInMinutes); - } - finally - { - clientThrottle.Release(); - } - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/README.md b/src/tasks/installer.tasks/BuildTools.Publish/README.md deleted file mode 100644 index 33a643dfab4d..000000000000 --- a/src/tasks/installer.tasks/BuildTools.Publish/README.md +++ /dev/null @@ -1,11 +0,0 @@ -The files in this directory are the closure of C# code from the BuildTools repo -that's necessary for the Core-Setup publish tasks. There are no changes except -automatically removing and sorting the using statements. - -Source: https://github.com/dotnet/buildtools/tree/55d43483866c7caeeace96355add3a9b12fa5795 - -Using the existing BuildTools code reduces the risk of behavior differences vs. -trying to find equivalents in Arcade. The upcoming new Arcade-powered publish -functionality makes short-term effort to deduplicate these tasks throwaway work. - -See [core-setup/#8285 "Migrate to Arcade's blob publish infrastructure"](https://github.com/dotnet/core-setup/issues/8285) diff --git a/src/tasks/installer.tasks/FinalizeBuild.cs b/src/tasks/installer.tasks/FinalizeBuild.cs deleted file mode 100644 index ee1fe8fcc630..000000000000 --- a/src/tasks/installer.tasks/FinalizeBuild.cs +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using Microsoft.DotNet.Build.CloudTestTasks; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class FinalizeBuild : AzureConnectionStringBuildTask - { - [Required] - public string SemaphoreBlob { get; set; } - [Required] - public string FinalizeContainer { get; set; } - public string MaxWait { get; set; } - public string Delay { get; set; } - [Required] - public string ContainerName { get; set; } - [Required] - public string Channel { get; set; } - [Required] - public string SharedFrameworkNugetVersion { get; set; } - [Required] - public string SharedHostNugetVersion { get; set; } - [Required] - public string ProductVersion { get; set; } - [Required] - public string Version { get; set; } - [Required] - public string CommitHash { get; set; } - public bool ForcePublish { get; set; } - - private Regex _versionRegex = new Regex(@"(?\d+\.\d+\.\d+)(-(?[^-]+-)?(?\d+)-(?\d+))?"); - - public override bool Execute() - { - ParseConnectionString(); - - if (Log.HasLoggedErrors) - { - return false; - } - - if (!FinalizeContainer.EndsWith("/")) - { - FinalizeContainer = $"{FinalizeContainer}/"; - } - string targetVersionFile = $"{FinalizeContainer}{Version}"; - - CreateBlobIfNotExists(SemaphoreBlob); - - AzureBlobLease blobLease = new AzureBlobLease(AccountName, AccountKey, ConnectionString, ContainerName, SemaphoreBlob, Log); - Log.LogMessage($"Acquiring lease on semaphore blob '{SemaphoreBlob}'"); - blobLease.Acquire(); - - // Prevent race conditions by dropping a version hint of what version this is. If we see this file - // and it is the same as our version then we know that a race happened where two+ builds finished - // at the same time and someone already took care of publishing and we have no work to do. - if (IsLatestSpecifiedVersion(targetVersionFile) && !ForcePublish) - { - Log.LogMessage(MessageImportance.Low, $"version hint file for publishing finalization is {targetVersionFile}"); - Log.LogMessage(MessageImportance.High, $"Version '{Version}' is already published, skipping finalization."); - Log.LogMessage($"Releasing lease on semaphore blob '{SemaphoreBlob}'"); - blobLease.Release(); - return true; - } - else - { - - // Delete old version files - GetBlobList(FinalizeContainer) - .Select(s => s.Replace("/dotnet/", "")) - .Where(w => _versionRegex.Replace(Path.GetFileName(w), "") == "") - .ToList() - .ForEach(f => TryDeleteBlob(f)); - - - // Drop the version file signaling such for any race-condition builds (see above comment). - CreateBlobIfNotExists(targetVersionFile); - - try - { - CopyBlobs($"Runtime/{ProductVersion}/", $"Runtime/{Channel}/"); - - // Generate the latest version text file - string sfxVersion = GetSharedFrameworkVersionFileContent(); - PublishStringToBlob(ContainerName, $"Runtime/{Channel}/latest.version", sfxVersion, "text/plain"); - } - finally - { - blobLease.Release(); - } - } - return !Log.HasLoggedErrors; - } - - private string GetSharedFrameworkVersionFileContent() - { - string returnString = $"{CommitHash}{Environment.NewLine}"; - returnString += $"{SharedFrameworkNugetVersion}{Environment.NewLine}"; - return returnString; - } - - public bool CopyBlobs(string sourceFolder, string destinationFolder) - { - bool returnStatus = true; - List> copyTasks = new List>(); - string[] blobs = GetBlobList(sourceFolder); - foreach (string blob in blobs) - { - string targetName = Path.GetFileName(blob) - .Replace(SharedFrameworkNugetVersion, "latest") - .Replace(SharedHostNugetVersion, "latest"); - string sourceBlob = blob.Replace($"/{ContainerName}/", ""); - string destinationBlob = $"{destinationFolder}{targetName}"; - Log.LogMessage($"Copying blob '{sourceBlob}' to '{destinationBlob}'"); - copyTasks.Add(CopyBlobAsync(sourceBlob, destinationBlob)); - } - Task.WaitAll(copyTasks.ToArray()); - copyTasks.ForEach(c => returnStatus &= c.Result); - return returnStatus; - } - - public bool TryDeleteBlob(string path) - { - return DeleteBlob(ContainerName, path); - } - - public void CreateBlobIfNotExists(string path) - { - var blobList = GetBlobList(path); - if(blobList.Count() == 0) - { - PublishStringToBlob(ContainerName, path, DateTime.Now.ToString()); - } - } - - public bool IsLatestSpecifiedVersion(string versionFile) - { - var blobList = GetBlobList(versionFile); - return blobList.Count() != 0; - } - - public bool DeleteBlob(string container, string blob) - { - return DeleteAzureBlob.Execute(AccountName, - AccountKey, - ConnectionString, - container, - blob, - BuildEngine, - HostObject); - } - - public Task CopyBlobAsync(string sourceBlobName, string destinationBlobName) - { - return CopyAzureBlobToBlob.ExecuteAsync(AccountName, - AccountKey, - ConnectionString, - ContainerName, - sourceBlobName, - destinationBlobName, - BuildEngine, - HostObject); - } - - public string[] GetBlobList(string path) - { - return ListAzureBlobs.Execute(AccountName, - AccountKey, - ConnectionString, - ContainerName, - path, - BuildEngine, - HostObject); - } - - public bool PublishStringToBlob(string container, string blob, string contents, string contentType = null) - { - return PublishStringToAzureBlob.Execute(AccountName, - AccountKey, - ConnectionString, - container, - blob, - contents, - contentType, - BuildEngine, - HostObject); - } - } -} diff --git a/src/tasks/installer.tasks/BuildTools.Publish/Tasks/GenerateChecksums.cs b/src/tasks/installer.tasks/GenerateChecksums.cs similarity index 100% rename from src/tasks/installer.tasks/BuildTools.Publish/Tasks/GenerateChecksums.cs rename to src/tasks/installer.tasks/GenerateChecksums.cs diff --git a/src/tasks/installer.tasks/installer.tasks.csproj b/src/tasks/installer.tasks/installer.tasks.csproj index 70b66fa53669..04f5997217ea 100644 --- a/src/tasks/installer.tasks/installer.tasks.csproj +++ b/src/tasks/installer.tasks/installer.tasks.csproj @@ -1,37 +1,24 @@ - netstandard2.0;net461 - false + $(NetCoreAppToolCurrent);net461 false false - - - - - - - - + - - - + - - - diff --git a/src/tasks/tasks.proj b/src/tasks/tasks.proj index 3477a1e9aee1..17ea46bc5866 100644 --- a/src/tasks/tasks.proj +++ b/src/tasks/tasks.proj @@ -19,7 +19,7 @@ locking issues vs. the persistent nodes that loaded the task DLL for the previous build. It isn't particularly accurate, but better than nothing. --> - @@ -27,10 +27,6 @@ - -