dotnet_runtime/eng/pipelines/installer/jobs/base-job.yml

525 lines
19 KiB
YAML

parameters:
buildConfig: ''
osGroup: ''
archType: ''
osSubgroup: ''
platform: ''
crossrootfsDir: ''
timeoutInMinutes: 120
condition: true
container: ''
buildSteps: []
dependsOn: []
variables: []
name: ''
displayName: ''
pool: ''
packageDistroList:
- image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-debpkg-e5cf912-20175003025046
imageRid: ubuntu.14.04
rids: []
- image: mcr.microsoft.com/dotnet-buildtools/prereqs:rhel-7-rpmpkg-c982313-20174116044113
imageRid: rhel.7
rids:
- centos.7
- fedora.27
- opensuse.42
- oraclelinux.7
- sles.12
isOfficialBuild: false
useOfficialAllConfigurations: false
buildFullPlatformManifest: false
liveRuntimeBuildConfig: ''
liveLibrariesBuildConfig: ''
runtimeFlavor: 'coreclr'
platforms: []
jobs:
- job: ${{ format('installer_{0}_{1}_{2}', parameters.runtimeFlavor, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}
displayName: ${{ format('Installer Build and Test {0} {1} {2}', parameters.runtimeFlavor, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}
condition: and(succeeded(), ${{ parameters.condition }})
pool: ${{ parameters.pool }}
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
# Do not attempt to clean workspace on Linux: the agent might not be able to remove the files
# because they may be owned by "root" due to the way this job uses Docker. We do our own cleanup
# in this case as a prepare step.
${{ if ne(parameters.osGroup, 'Linux') }}:
workspace:
clean: all
variables:
- ${{ each variable in parameters.variables }}:
- ${{ variable }}
- name: OfficialBuildArg
value: ''
- name: _BuildConfig
value: ${{ parameters.buildConfig }}
- name: SkipTests
value: ${{ or(
not(in(parameters.archType, 'x64', 'x86')),
eq(parameters.runtimeFlavor, 'mono'),
eq(parameters.isOfficialBuild, true),
ne(parameters.crossrootfsDir, '')) }}
- name: SignType
value: test
# Set up non-PR build from internal project
- ${{ if eq(parameters.isOfficialBuild, true) }}:
- name: SignType
value: $[ coalesce(variables.OfficialSignType, 'real') ]
- name: OfficialBuildArg
value: /p:OfficialBuildId=$(Build.BuildNumber)
- name: buildCommandSourcesDirectory
${{ if not(in(parameters.osGroup, 'Linux', 'FreeBSD')) }}:
value: '$(Build.SourcesDirectory)/'
# This job runs within Docker containers, so Build.SourcesDirectory is not accurate.
${{ if in(parameters.osGroup, 'Linux', 'FreeBSD') }}:
value: '/root/runtime/'
###
### Platform-specific variable setup
###
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- name: CommonMSBuildArgs
value: >-
/p:TargetArchitecture=${{ parameters.archType }}
/p:PortableBuild=true
/p:SkipTests=$(SkipTests)
/p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
$(OfficialBuildArg)
- name: MsbuildSigningArguments
value: >-
/p:CertificateId=400
/p:DotNetSignType=$(SignType)
- name: TargetArchitecture
value: ${{ parameters.archType }}
- name: BaseJobBuildCommand
value: >-
build.cmd -subset installer -test -ci
-configuration $(_BuildConfig)
$(LiveOverridePathArgs)
$(CommonMSBuildArgs)
$(MsbuildSigningArguments)
- ${{ if eq(parameters.osGroup, 'OSX') }}:
- name: CommonMSBuildArgs
value: >-
/p:PortableBuild=true
/p:SkipTests=$(SkipTests)
/p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
- name: BaseJobBuildCommand
value: >-
$(Build.SourcesDirectory)/build.sh -subset installer -test -ci
-configuration $(_BuildConfig)
$(LiveOverridePathArgs)
$(CommonMSBuildArgs)
$(OfficialBuildArg)
- ${{ if in(parameters.osGroup, 'iOS', 'tvOS', 'Android', 'Browser') }}:
- name: CommonMSBuildArgs
value: >-
/p:PortableBuild=true
/p:SkipTests=$(SkipTests)
- name: BaseJobBuildCommand
value: >-
$(Build.SourcesDirectory)/build.sh -subset installer --ci --test
-configuration $(_BuildConfig)
-os ${{ parameters.osGroup }}
-arch ${{ parameters.archType }}
/p:StripSymbols=true
$(LiveOverridePathArgs)
$(CommonMSBuildArgs)
$(OfficialBuildArg)
- ${{ if in(parameters.osGroup, 'Linux', 'FreeBSD') }}:
# Preserve the NuGet authentication env vars into the Docker container.
# The 'NuGetAuthenticate' build step may have set these.
- name: PreserveNuGetAuthDockerArgs
value: >-
-e VSS_NUGET_URI_PREFIXES
-e VSS_NUGET_ACCESSTOKEN
- ${{ if ne(parameters.container, '') }}:
- name: RunArguments
value: >-
docker run --privileged --rm
-v "$(Build.SourcesDirectory):/root/runtime"
-w="/root/runtime"
$(PreserveNuGetAuthDockerArgs)
-e ROOTFS_DIR=${{ parameters.crossrootfsDir }}
${{ parameters.container }}
- name: BuildScript
value: ./build.sh
- name: MSBuildScript
value: /root/runtime/eng/common/msbuild.sh
- ${{ if eq(parameters.isOfficialBuild, true) }}:
- name: BuildScript
value: ./eng/install-nuget-credprovider-then-build.sh --subset installer
- name: MSBuildScript
value: /root/runtime/eng/install-nuget-credprovider-then-msbuild.sh
- name: CommonMSBuildArgs
value: >-
/p:Configuration=$(_BuildConfig)
/p:TargetOS=${{ parameters.osGroup }}
/p:TargetArchitecture=${{ parameters.archType }}
/p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
$(OfficialBuildArg)
- name: _PortableBuild
value: ${{ eq(parameters.osSubgroup, '') }}
- ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'Linux')) }}:
# Set output RID manually: musl isn't properly detected. Make sure to also convert linux to
# lowercase for RID format. (Detection normally converts, but we're preventing it.)
- name: OutputRidArg
value: /p:OutputRid=linux-musl-${{ parameters.archType }}
- name: _PortableBuild
value: true
- name: BuildArguments
value: >-
-subset installer -test -ci
/p:CrossBuild=${{ ne(parameters.crossrootfsDir, '') }}
/p:PortableBuild=$(_PortableBuild)
/p:SkipTests=$(SkipTests)
$(LiveOverridePathArgs)
$(CommonMSBuildArgs)
$(OutputRidArg)
- name: PublishArguments
value: >-
/p:PortableBuild=$(_PortableBuild)
$(CommonMSBuildArgs)
$(OutputRidArg)
/bl:msbuild.publish.binlog
# Tell the build to package up the bits from the portable build.
- name: PackagePortableBitsArgs
value: >-
/p:PortableBuild=false
/p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/
/p:InstallerSourceOSPlatformConfig=linux-x64.$(_BuildConfig)
- name: DockerRunMSBuild
value: >-
docker run
-v $(Build.SourcesDirectory):/root/runtime
-v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/
-w=/root/runtime
$(PreserveNuGetAuthDockerArgs)
- name: installersSubsetArg
value: --subset Installers
- name: BaseJobBuildCommand
value: |
set -x
df -h
$(RunArguments) $(BuildScript) $(BuildArguments)
###
### Common Live build override variable setup
###
- name: LiveOverridePathArgs
value: >-
$(RuntimeArtifactsArgs)
$(LibrariesConfigurationArg)
$(LibrariesAllConfigurationsOverridePathArg)
$(AllArtifactsArgs)
- name: RuntimeArtifactsArgs
value: ''
- name: LibrariesConfigurationArg
value: ''
- name: LibrariesAllConfigurationsOverridePathArg
value: ''
- name: AllArtifactsArgs
value: ''
- name: RuntimeDownloadPath
value: ''
- name: LibrariesDownloadPath
value: ''
- name: LibrariesDownloadPathAllConfigurations
value: ''
- name: AllArtifactsDownloadPath
value: ''
- ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
- name: liveRuntimeLegName
value: ${{ format('{0}{1}_{2}_{3}',
parameters.osGroup,
parameters.osSubgroup,
parameters.archType,
parameters.liveRuntimeBuildConfig) }}
- name: RuntimeDownloadPath
value: 'artifacts/transport/${{ parameters.runtimeFlavor }}'
- name: RuntimeArtifactsArgs
value: >-
/p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath)
/p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }}
- name: RuntimeArtifactName
value: $(runtimeFlavorName)Product_$(liveRuntimeLegName)
- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
- name: liveLibrariesLegName
value: ${{ format('{0}{1}_{2}_{3}',
parameters.osGroup,
parameters.osSubgroup,
parameters.archType,
parameters.liveLibrariesBuildConfig) }}
- name: LibrariesDownloadPath
value: 'artifacts'
- name: LibrariesArtifactName
value: libraries_bin_$(liveLibrariesLegName)
- name: LibrariesConfigurationArg
value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}'
# Download allconfigurations bits to a separate location. It overlaps with the live libraries
# download and causes conflict when unzipping to the same location.
- ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
- name: LibrariesDownloadPathAllConfigurations
value: 'artifacts/transport/librariesallconfigurations'
- name: LibrariesAllConfigurationsOverridePathArg
value: /p:LibrariesAllConfigurationsOverridePath=${{ parameters.buildCommandSourcesDirectory }}$(LibrariesDownloadPathAllConfigurations)
- name: LibrariesArtifactNameAllConfigurations
value: libraries_bin_official_allconfigurations
# Download all upstream platforms and incorporate in the platform manifest.
- ${{ if eq(parameters.buildFullPlatformManifest, true) }}:
- name: AllArtifactsDownloadPath
value: 'artifacts/transport/AllArtifacts'
- name: AllArtifactsArgs
value: >-
/p:AllArtifactsDownloadPath=${{ parameters.buildCommandSourcesDirectory }}$(AllArtifactsDownloadPath)
/p:BuildFullPlatformManifest=true
dependsOn:
- checkout
- ${{ parameters.dependsOn }}
- ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
- ${{ format('{0}_product_build_{1}{2}_{3}_{4}',
parameters.runtimeFlavor,
parameters.osGroup,
parameters.osSubgroup,
parameters.archType,
parameters.liveRuntimeBuildConfig) }}
- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
- libraries_build_${{ format('{0}{1}_{2}_{3}',
parameters.osGroup,
parameters.osSubgroup,
parameters.archType,
parameters.liveLibrariesBuildConfig) }}
- ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
- libraries_build_allconfigurations_Windows_NT_x64_Release
- ${{ if eq(parameters.buildFullPlatformManifest, true) }}:
- ${{ each platform in parameters.platforms }}:
- ${{ parameters.runtimeFlavor }}_product_build_${{ platform }}_${{ parameters.liveRuntimeBuildConfig }}
- libraries_build_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}
steps:
- ${{ if ne(parameters.container, '') }}:
# Builds don't set user ID, so files might be owned by root and unable to be cleaned up by AzDO.
# Clean up the build dirs ourselves in another Docker container to avoid failures.
# Using hosted agents is tracked by https://github.com/dotnet/core-setup/issues/4997
- script: |
set -x
docker run --rm \
-v "$(Agent.BuildDirectory):/root/build" \
-w /root/build \
${{ parameters.container }} \
bash -c '
rm -v -rf a b s'
mkdir "$(Agent.BuildDirectory)/s"
displayName: Clean up old artifacts owned by root
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
# NuGet's http cache lasts 30 minutes. If we're on a static machine, this may interfere with
# auto-update PRs by preventing the CI build from fetching the new version. Delete the cache.
- powershell: Remove-Item -Recurse -ErrorAction Ignore "$env:LocalAppData\NuGet\v3-cache"
displayName: Clear NuGet http cache (if exists)
- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin for Signing
inputs:
signType: $(SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
continueOnError: false
condition: and(succeeded(), in(variables['SignType'], 'real', 'test'))
- template: /eng/pipelines/common/clone-checkout-bundle-step.yml
# Download and extract artifacts for earlier subsets.
- task: DownloadBuildArtifacts@0
displayName: 'Download artifacts for all platforms'
inputs:
buildType: current
downloadType: specific
downloadPath: '$(Build.SourcesDirectory)/__download__/AllPlatforms/'
itemPattern: |
$(runtimeFlavorName)Product_*/**
libraries_bin_*/**
- ${{ if eq(parameters.buildFullPlatformManifest, true) }}:
- ${{ each platform in parameters.platforms }}:
- task: ExtractFiles@1
displayName: 'Unzip $(runtimeFlavorName) artifacts: ${{ platform }}'
inputs:
archiveFilePatterns: |
$(Build.SourcesDirectory)/__download__/AllPlatforms/$(runtimeFlavorName)Product_${{ platform }}_${{ parameters.liveRuntimeBuildConfig }}.*
destinationFolder: $(AllArtifactsDownloadPath)/$(runtimeFlavorName)Product_${{ platform }}_${{ parameters.liveRuntimeBuildConfig }}/
cleanUnpackFolder: false
- task: ExtractFiles@1
displayName: 'Unzip Libraries artifacts: ${{ platform }}'
inputs:
archiveFilePatterns: |
$(Build.SourcesDirectory)/__download__/AllPlatforms/libraries_bin_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}.*
destinationFolder: $(AllArtifactsDownloadPath)/libraries_bin_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}/
cleanUnpackFolder: false
- ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
unpackFolder: $(Build.SourcesDirectory)/$(RuntimeDownloadPath)
artifactFileName: '$(RuntimeArtifactName)$(archiveExtension)'
artifactName: '$(RuntimeArtifactName)'
displayName: '$(runtimeFlavorName) artifacts'
- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPath)
artifactFileName: '$(LibrariesArtifactName)$(archiveExtension)'
artifactName: '$(LibrariesArtifactName)'
displayName: 'Libraries artifacts'
cleanUnpackFolder: false
- ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPathAllConfigurations)
# Always use '.zip' because AllConfigurations is built on Windows but used everywhere.
artifactFileName: '$(LibrariesArtifactNameAllConfigurations).zip'
artifactName: '$(LibrariesArtifactNameAllConfigurations)'
displayName: 'Libraries artifacts (AllConfigurations)'
cleanUnpackFolder: false
- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- script: |
du -sh $(Build.SourcesDirectory)/*
df -h
displayName: Disk Usage before Build
- script: $(BaseJobBuildCommand)
displayName: Build
- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- script: |
du -sh $(Build.SourcesDirectory)/*
df -h
displayName: Disk Usage after Build
# Only in glibc leg, we produce RPMs and Debs
- ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.platform, 'Linux_x64'), eq(parameters.osSubgroup, ''))}}:
- task: CopyFiles@2
displayName: 'Copy built Portable linux-x64 binaries to staging directory'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/obj/linux-x64.$(_BuildConfig)/sharedFrameworkPublish'
TargetFolder: '$(Build.StagingDirectory)/sharedFrameworkPublish'
- ${{ each packageBuild in parameters.packageDistroList }}:
# This leg's RID matches the build image. Build its distro-dependent packages, as well as
# the distro-independent installers. (There's no particular reason to build the distro-
# independent installers on this leg, but we need to do it somewhere.)
- template: steps/build-linux-package.yml
parameters:
distroRid: ${{ packageBuild.imageRid }}
image: ${{ packageBuild.image }}
packageStepDescription: Runtime Deps, Runtime, Framework Packs installers
subsetArg: $(installersSubsetArg)
- ${{ each rid in packageBuild.rids }}:
# Build distro-dependent packages.
- template: steps/build-linux-package.yml
parameters:
distroRid: ${{ rid }}
image: ${{ packageBuild.image }}
outputRidArg: /p:OutputRid=${{ rid }}-${{ parameters.archType }}
packageStepDescription: Runtime Deps installers
packagingArgs: /p:BuildDistroIndependentInstallers=false
subsetArg: $(installersSubsetArg)
- ${{ if ne(parameters.container, '') }}:
# Files may be owned by root because builds don't set user ID. Later build steps run 'find' in
# the source tree, which fails due to permissions in the 'NetCore*-Int-Pool' queues. This step
# prevents the failure by using chown to clean up our source tree.
- script: |
set -x
docker run --rm \
-v "$(Agent.BuildDirectory):/root/build" \
-w /root/build \
${{ parameters.container }} \
bash -c "chown -R $(id -u):$(id -g) *"
displayName: Update file ownership from root to build agent account
continueOnError: true
condition: succeededOrFailed()
- ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(parameters.isOfficialBuild, true)) }}:
- task: NuGetCommand@2
displayName: Push Visual Studio NuPkgs
inputs:
command: push
packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/*/VS.Redist.Common.*.nupkg'
nuGetFeedType: external
publishFeedCredentials: 'DevDiv - VS package feed'
condition: and(
succeeded(),
eq(variables['_BuildConfig'], 'Release'),
ne(variables['DisableVSPublish'], 'true'))
- template: steps/upload-job-artifacts.yml
parameters:
name: ${{ coalesce(parameters.name, parameters.platform) }}
runtimeFlavor: ${{ parameters.runtimeFlavor }}
skipTests: $(SkipTests)
isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }}
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- script: set -x && df -h
displayName: Check remaining storage space
condition: always()
continueOnError: true
# Force clean up machine in case any docker images are left behind
- ${{ if ne(parameters.container, '') }}:
- script: docker system prune -af && df -h
displayName: Run Docker clean up
condition: succeededOrFailed()