Bringing Fairlearn - GridSearch to ML.NET #6279
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
| | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Text; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.ML.AutoML; | ||
| using Microsoft.ML.Fairlearn.reductions; | ||
| | ||
| namespace Microsoft.ML.Fairlearn.AutoML | ||
| { | ||
| /// <summary> | ||
| /// An internal class that holds the gridLimit value to conduct gridsearch. | ||
| /// Needed to pass the value into the AutoMLExperiment as a singleton | ||
| /// </summary> | ||
| internal class GridLimit | ||
| { | ||
| public float Value { get; set; } | ||
| } | ||
| /// <summary> | ||
| /// An extension class used to add more options to the Fairlearn girdsearch experiment | ||
| /// </summary> | ||
| public static class AutoMLExperimentExtension | ||
| { | ||
| public static AutoMLExperiment SetBinaryClassificationMoment(this AutoMLExperiment experiment, ClassificationMoment moment) | ||
| { | ||
| experiment.ServiceCollection.AddSingleton(moment); | ||
| experiment.SetTunerFactory<CostFrugalWithLambdaTunerFactory>(); | ||
| | ||
| return experiment; | ||
| } | ||
| | ||
| public static AutoMLExperiment SetGridLimit(this AutoMLExperiment experiment, float gridLimit) | ||
| Member There was a problem hiding this comment. Other than
| ||
| { | ||
| var gridLimitObject = new GridLimit(); | ||
| gridLimitObject.Value = gridLimit; | ||
| experiment.ServiceCollection.AddSingleton(gridLimitObject); | ||
| experiment.SetTunerFactory<CostFrugalWithLambdaTunerFactory>(); | ||
| | ||
| return experiment; | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
| | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Text; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.ML; | ||
| using Microsoft.ML.AutoML; | ||
| using Microsoft.ML.Fairlearn.reductions; | ||
| | ||
| namespace Microsoft.ML.Fairlearn.AutoML | ||
| { | ||
| internal class CostFrugalWithLambdaTunerFactory : ITunerFactory | ||
| { | ||
| private readonly IServiceProvider _provider; | ||
| private readonly ClassificationMoment _moment; | ||
| private readonly MLContext _context; | ||
| private readonly float _gridLimit = 10f; | ||
| Contributor There was a problem hiding this comment. Is there a need to set this when you override it in the constructor on line 27? | ||
| | ||
| public CostFrugalWithLambdaTunerFactory(IServiceProvider provider) | ||
| { | ||
| _provider = provider; | ||
| _moment = provider.GetService<ClassificationMoment>(); | ||
| _context = provider.GetService<MLContext>(); | ||
| _gridLimit = provider.GetService<GridLimit>().Value; | ||
| } | ||
| | ||
| public ITuner CreateTuner(TrialSettings settings) | ||
| { | ||
| var experimentSetting = _provider.GetService<AutoMLExperiment.AutoMLExperimentSettings>(); | ||
| var searchSpace = settings.Pipeline.SearchSpace; | ||
| var isMaximize = experimentSetting.IsMaximizeMetric; | ||
| | ||
| var lambdaSearchSpace = Utilities.GenerateBinaryClassificationLambdaSearchSpace(_context, _moment, gridLimit: _gridLimit); | ||
| searchSpace["_lambda_search_space"] = lambdaSearchSpace; | ||
| var initParameter = searchSpace.SampleFromFeatureSpace(searchSpace.Default); | ||
| Contributor There was a problem hiding this comment. Do you need this line? Doesn't look like | ||
| | ||
| return new RandomSearchTuner(searchSpace); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
| | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Text; | ||
| using Microsoft.ML.AutoML; | ||
| | ||
| namespace Microsoft.ML.Fairlearn | ||
| { | ||
| public sealed class FairlearnCatalog | ||
| { | ||
| private readonly MLContext _context; | ||
| public FairlearnMetricCatalog Metric; | ||
| | ||
| internal FairlearnCatalog(MLContext context) | ||
| { | ||
| this._context = context; | ||
| this.Metric = new FairlearnMetricCatalog(context); | ||
| } | ||
| | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
| using Microsoft.ML; | ||
| Contributor There was a problem hiding this comment. nit: space between comment and first using. | ||
| | ||
| namespace Microsoft.ML.Fairlearn | ||
| { | ||
| /// <summary> | ||
| /// Class containing AutoML extension methods to <see cref="MLContext"/> | ||
| /// </summary> | ||
| public static class MLContextExtension | ||
| { | ||
| /// <summary> | ||
| /// Returns a catalog of all possible Fairlearn operations. | ||
| /// </summary> | ||
| /// <param name="mlContext"><see cref="MLContext"/> instance.</param> | ||
| /// <returns>A catalog of all possible AutoML operations.</returns> | ||
| public static FairlearnCatalog Fairlearn(this MLContext mlContext) | ||
| { | ||
| return new FairlearnCatalog(mlContext); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| Contributor There was a problem hiding this comment. Have you tested building the nuget and using it just to make sure it works? | ||
| <Import Project="$(RepoRoot)eng/pkg/Pack.props" /> | ||
| | ||
| <PropertyGroup> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| <IncludeInPackage>Microsoft.ML.Fairlearn</IncludeInPackage> | ||
| </PropertyGroup> | ||
| | ||
| <ItemGroup> | ||
| <ProjectReference Include="..\Microsoft.Data.Analysis\Microsoft.Data.Analysis.csproj" /> | ||
| <ProjectReference Include="..\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj" /> | ||
| <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| </ProjectReference> | ||
| <ProjectReference Include="..\Microsoft.ML.SearchSpace\Microsoft.ML.SearchSpace.csproj"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| <IncludeInNuget>true</IncludeInNuget> | ||
| </ProjectReference> | ||
| <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" /> | ||
| | ||
| <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| </ProjectReference> | ||
| | ||
| </ItemGroup> | ||
| | ||
| </Project> | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit. This suggestion is invalid because no changes were made to the code. Suggestions cannot be applied while the pull request is closed. Suggestions cannot be applied while viewing a subset of changes. Only one suggestion per line can be applied in a batch. Add this suggestion to a batch that can be applied as a single commit. Applying suggestions on deleted lines is not supported. You must change the existing code in this line in order to create a valid suggestion. Outdated suggestions cannot be applied. This suggestion has been applied or marked resolved. Suggestions cannot be applied from pending reviews. Suggestions cannot be applied on multi-line comments. Suggestions cannot be applied while the pull request is queued to merge. Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We need this and the associated Release/Debug info back for FSharp Tests