fix official builds

Author: frank-dong-ms

build/publish.proj

@@ -1,23 +1,28 @@
 <Project>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.DotNet.BuildTools" Version="$(BuildToolsPackageVersion)" />
+ </ItemGroup>
+ 
  <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
 
- <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll" />
+ <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(NuGetPackageRoot)\microsoft.dotnet.buildtools\$(BuildToolsPackageVersion)\lib\Microsoft.DotNet.Build.Tasks.dll" />
 
  <PropertyGroup>
  <PublishSymbolsPackage>Microsoft.SymbolUploader.Build.Task</PublishSymbolsPackage>
  <EnablePublishSymbols Condition="'$(EnablePublishSymbols)'==''" >true</EnablePublishSymbols>
  <NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
  </PropertyGroup>
 
- <Import Project="$(PackagesDir)\$(PublishSymbolsPackage.ToLower())\$(PublishSymbolsPackageVersion)\build\PublishSymbols.targets" />
+ <Import Project="$(NuGetPackageRoot)\$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />
 
  <Target Name="PublishPackages">
  <Error Condition="'$(NuGetFeedUrl)' == ''" Text="Missing required property NuGetFeedUrl" />
  <Error Condition="'$(NuGetApiKey)' == ''" Text="Missing required property NuGetApiKey" />
 
  <ItemGroup>
- <NuGetPackages Include="$(PackageOutputPath)**\*.nupkg"
- Exclude="$(PackageOutputPath)**\*.symbols.*nupkg" />
+ <NuGetPackages Include="$(ArtifactsDir)packages\**\*.nupkg"
+ Exclude="$(ArtifactsDir)packages\**\*.snupkg" />
 
  <!--
  IgnorableErrorMessages applies to the "ExectWithRetriesForNuGetPush" task.
@@ -34,7 +39,7 @@
  <Message Text="Pushing ML.NET packages to $(NuGetFeedUrl)" />
 
  <PropertyGroup>
- <DotnetToolCommand>$(ToolsDir)dotnetcli/dotnet</DotnetToolCommand>
+ <DotnetToolCommand>$(MSBuildThisFileDirectory)..\.dotnet\dotnet</DotnetToolCommand>
  <NuGetPushCommand>$(DotnetToolCommand) nuget push --source $(NuGetFeedUrl) --api-key $(NuGetApiKey) --timeout $(NuGetPushTimeoutSeconds)</NuGetPushCommand>
  </PropertyGroup>
 
@@ -46,7 +51,7 @@
  Condition="'$(EnablePublishSymbols)'=='true'"
  DependsOnTargets="SetupPublishSymbols">
  <Message Text="Attempting to Publish Symbols" />
- <Error Condition="!Exists('$(PackageOutputPath)')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
+ <Error Condition="!Exists('$(ArtifactsDir)packages')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
  <Error Condition="'$(SymbolServerPath)'==''" Text="Missing property SymbolServerPath" />
  <Error Condition="'$(SymbolServerPAT)'==''" Text="Missing property SymbolServerPAT" />
  <CallTarget Targets="PublishSymbols" />
@@ -59,7 +64,7 @@
  </PropertyGroup>
 
  <ItemGroup>
- <SymbolPackagesToPublish Include="$(PackageOutputPath)**\*.symbols.*nupkg" />
+ <SymbolPackagesToPublish Include="$(ArtifactsDir)packages\**\*.snupkg" />
  </ItemGroup>
 
  <Message Importance="High" Text="Publishing @(SymbolPackagesToPublish) to $(SymbolServerPath)"/>

build/vsts-ci.yml

@@ -23,14 +23,18 @@ phases:
  - agent.os -equals linux
  container: CentosContainer
  steps:
+ - script: ./restore.sh
+ displayName: restore all projects
+ - script: ./build.sh -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+ displayName: build redist
  # Only build native assets to avoid conflicts.
- - script: ./build.sh -buildNative -$(BuildConfig) -skipRIDAgnosticAssets
+ - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64
  displayName: Build
 
  - task: PublishBuildArtifacts@1
  displayName: Publish Linux package assets
  inputs:
- pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+ pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
  artifactName: PackageAssets
  artifactType: container
 
@@ -46,16 +50,26 @@ phases:
  queue:
  name: Hosted macOS
  steps:
- - script: brew update && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
+ - script: |
+ brew uninstall openssl@1.0.2t |
+ brew uninstall python@2.7.17 |
+ brew untap local/openssl |
+ brew untap local/python2
+ displayName: MacOS Homebrew bug Workaround
+ - script: brew update && brew unlink python@3.8 && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
  displayName: Install build dependencies
+ - script: ./restore.sh
+ displayName: restore all projects
+ - script: ./build.sh -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+ displayName: build redist
  # Only build native assets to avoid conflicts.
- - script: ./build.sh -buildNative -$(BuildConfig) -skipRIDAgnosticAssets
+ - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64
  displayName: Build
 
  - task: PublishBuildArtifacts@1
  displayName: Publish macOS package assets
  inputs:
- pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+ pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
  artifactName: PackageAssets
  artifactType: container
 
@@ -88,27 +102,26 @@ phases:
  continueOnError: false
  condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
 
+ - script: ./restore.cmd
+ displayName: restore all projects
+ - script: ./build.cmd -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+ displayName: build redist
  # Only build native assets to avoid conflicts.
- - script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets
+ - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86
  displayName: Build
 
- - task: MSBuild@1
- displayName: Sign Windows_x86 Binaries
- inputs:
- solution: build/sign.proj
- msbuildArguments: /p:SignType=$(_SignType)
- msbuildVersion: 15.0
- continueOnError: false
+ - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:SignBinaries=true
+ displayName: sign binaries
 
  - task: PublishBuildArtifacts@1
  displayName: Publish Windows_x86 package assets
  inputs:
- pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+ pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
  artifactName: PackageAssets
  artifactType: container
 
  # Terminate all dotnet build processes.
- - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+ - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
  displayName: Dotnet Server Shutdown
 
 ################################################################################
@@ -141,7 +154,7 @@ phases:
  condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
 
  # Build both native and managed assets.
- - script: ./build.cmd -$(BuildConfig)
+ - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64
  displayName: Build
 
  - task: ComponentGovernanceComponentDetection@0
@@ -151,23 +164,18 @@ phases:
  alertWarningLevel: 'High'
 
 
- - task: MSBuild@1
- displayName: Sign Windows_x64 Binaries
- inputs:
- solution: build/sign.proj
- msbuildArguments: /p:SignType=$(_SignType)
- msbuildVersion: 15.0
- continueOnError: false
+ - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:SignBinaries=true
+ displayName: sign binaries
 
  - task: PublishBuildArtifacts@1
  displayName: Publish Windows_x64 package assets
  inputs:
- pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+ pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
  artifactName: PackageAssets
  artifactType: container
 
  # Terminate all dotnet build processes.
- - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+ - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
  displayName: Dotnet Server Shutdown
 
 ################################################################################
@@ -213,31 +221,27 @@ phases:
  displayName: Download package assets
  inputs:
  artifactName: PackageAssets
- downloadPath: $(Build.SourcesDirectory)/bin/obj/packages
+ downloadPath: $(Build.SourcesDirectory)/artifacts/pkgassets
 
  # Workaround https://github.com/Microsoft/vsts-tasks/issues/6739
  - task: CopyFiles@2
  displayName: Copy package assets to correct folder
  inputs:
- sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
- targetFolder: $(Build.SourcesDirectory)/bin/obj/packages
+ sourceFolder: $(Build.SourcesDirectory)/artifacts/pkgassets/PackageAssets
+ targetFolder: $(Build.SourcesDirectory)/artifacts/pkgassets
 
- - script: ./build.cmd -buildPackages
- displayName: Create Packages
+ - script: ./build.cmd -pack -configuration $(BuildConfig)
+ displayName: Build Packages
 
- - task: MSBuild@1
- displayName: Sign Packages
- inputs:
- solution: build/sign.proj
- msbuildArguments: /p:SignType=$(_SignType) /p:SignNugetPackages=true
- msbuildVersion: 15.0
+ - script: ./sign.cmd /p:SignNugetPackages=true
+ displayName: sign packages
  continueOnError: false
 
  - task: NuGetAuthenticate@0
  inputs:
  nuGetServiceConnections: machinelearning-dnceng-public-feed # To allow publishing to a feed of another organization
 
- - script: Tools\dotnetcli\dotnet msbuild build\publish.proj /t:PublishPackages /p:NuGetFeedUrl=$(_AzureDevopsFeedUrl) /p:NuGetApiKey=AzureArtifacts
+ - script: $(Build.SourcesDirectory)\.dotnet\dotnet.exe msbuild build\publish.proj /t:PublishPackages /p:NuGetFeedUrl=$(_AzureDevopsFeedUrl) /p:NuGetApiKey=AzureArtifacts
  displayName: Publish Packages to AzureDevOps Feed
 
  - task: MSBuild@1
@@ -257,5 +261,5 @@ phases:
  continueOnError: true
 
  # Terminate all dotnet build processes.
- - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+ - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
  displayName: Dotnet Server Shutdown

eng/Signing.props

@@ -1,6 +1,14 @@
 <Project>
- <ItemGroup>
- <!-- <StrongNameSignInfo Include="$(RepoRoot)\Tools\Open.snk" PublicKeyToken="cc7b13ffcd2ddd51" CertificateName="Microsoft400" />
- <StrongNameSignInfo Include="$(RepoRoot)\Tools\Test.snk" PublicKeyToken="9d77cc7ad39b68eb" CertificateName="Microsoft400" /> -->
- </ItemGroup
+ <ItemGroup Condition="'$(SignBinaries)' == 'true'">
+ <ItemsToSign Remove="@(ItemsToSign)" />
+ <ItemsToSign Include="$(ArtifactsDir)pkgassets\**\*.dll" />
+ <FileExtensionSignInfo Include="*.dll" CertificateName="Microsoft400" />
+ </ItemGroup>
+ <ItemGroup Condition="'$(SignNugetPackages)' == 'true'">
+ <ItemsToSign Remove="@(ItemsToSign)" />
+ <ItemsToSign Include="$(ArtifactsDir)packages\**\*.nupkg" />
+ <ItemsToSign Include="$(ArtifactsDir)packages\**\*.snupkg" />
+ <FileExtensionSignInfo Include="*.nupkg" CertificateName="NuGet" />
+ <FileExtensionSignInfo Include="*.snupkg" CertificateName="NuGet" />
+ </ItemGroup>
 </Project>

eng/Versions.props

@@ -37,6 +37,7 @@
  <ReportGeneratorVersion>4.3.6</ReportGeneratorVersion>
  <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.19225.5</MicrosoftDotNetApiCompatPackageVersion>
  <MicrosoftSourceLinkVersion>1.1.0-beta-20206-02</MicrosoftSourceLinkVersion>
+ <BuildToolsPackageVersion>3.0.0-preview4-04926-01</BuildToolsPackageVersion>
 
  <!-- Test-only Dependencies -->
  <BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>

restore.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -warnAsError 0 %*"
+exit /b %ErrorLevel%

restore.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+"$DIR/eng/common/build.sh" --restore --warnAsError false "$@"

sign.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -sign -warnAsError 0 %*"
+exit /b %ErrorLevel%

src/Microsoft.Extensions.ML/Microsoft.Extensions.ML.csproj

@@ -3,6 +3,7 @@
 
  <PropertyGroup>
  <TargetFramework>netstandard2.0</TargetFramework>
+ <IncludeInPackage>Microsoft.Extensions.ML</IncludeInPackage>
  <PackageDescription>An integration package for ML.NET models on scalable web apps and services.</PackageDescription>
  </PropertyGroup>
 

src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

@@ -3,6 +3,7 @@
 
  <PropertyGroup>
  <TargetFramework>netstandard2.0</TargetFramework>
+ <IncludeInPackage>Microsoft.ML.AutoML</IncludeInPackage>
  <PackageDescription>ML.NET AutoML: Optimizes an ML pipeline for your dataset, by automatically locating the best feature engineering, model, and hyperparameters</PackageDescription>
  </PropertyGroup>
 

src/Microsoft.ML.CodeGenerator/Microsoft.ML.CodeGenerator.csproj

@@ -3,6 +3,7 @@
 
  <PropertyGroup>
  <TargetFramework>netstandard2.0</TargetFramework>
+ <IncludeInPackage>Microsoft.ML.CodeGenerator</IncludeInPackage>
  <PackageDescription>ML.NET Code Generator</PackageDescription>
  </PropertyGroup>