Merge pull request #1664 from IdsTeepe/feat/dotnet-based-nuget-query

Feat: Dynamically get query service from nuget package source

Author: arturcic

dist/tools/libs/tools.mjs

@@ -4,6 +4,22 @@ import * as os from 'node:os';
 import * as path from 'node:path';
 import { s as semverExports } from './semver.mjs';
 
+var NugetServiceType = /* @__PURE__ */ ((NugetServiceType2) => {
+ NugetServiceType2["Catalog"] = "Catalog";
+ NugetServiceType2["PackageBaseAddress"] = "PackageBaseAddress";
+ NugetServiceType2["PackageDetailsUriTemplate"] = "PackageDetailsUriTemplate";
+ NugetServiceType2["PackagePublish"] = "PackagePublish";
+ NugetServiceType2["ReadmeUriTemplate"] = "ReadmeUriTemplate";
+ NugetServiceType2["RegistrationsBaseUrl"] = "RegistrationsBaseUrl";
+ NugetServiceType2["ReportAbuseUriTemplate"] = "ReportAbuseUriTemplate";
+ NugetServiceType2["RepositorySignatures"] = "RepositorySignatures";
+ NugetServiceType2["SearchAutocompleteService"] = "SearchAutocompleteService";
+ NugetServiceType2["SearchQueryService"] = "SearchQueryService";
+ NugetServiceType2["SymbolPackagePublish"] = "SymbolPackagePublish";
+ NugetServiceType2["VulnerabilityInfo"] = "VulnerabilityInfo";
+ return NugetServiceType2;
+})(NugetServiceType || {});
+
 class ArgumentsBuilder {
  args = [];
  isWindows = os.platform() === "win32";
@@ -167,7 +183,6 @@ class DotnetTool {
  constructor(buildAgent) {
  this.buildAgent = buildAgent;
  }
- static nugetRoot = "https://azuresearch-usnc.nuget.org/query";
  disableTelemetry() {
  this.buildAgent.info("Disable Telemetry");
  this.buildAgent.setVariable("DOTNET_CLI_TELEMETRY_OPTOUT", "true");
@@ -318,23 +333,58 @@ class DotnetTool {
  }
  return path.normalize(workDir);
  }
- async queryLatestMatch(toolName, versionSpec, includePrerelease) {
- this.buildAgent.info(
- `Querying tool versions for ${toolName}${versionSpec ? `@${versionSpec}` : ""} ${includePrerelease ? "including pre-releases" : ""}`
- );
+ async getQueryServices() {
+ const builder = new ArgumentsBuilder().addArgument("nuget").addArgument("list").addArgument("source").addKeyValue("format", "short");
+ const result = await this.execute("dotnet", builder.build());
+ const nugetSources = [
+ ...(result.stdout ?? "").matchAll(/^E (?<index>.+)/gm)
+ ].map((m) => m.groups.index);
+ if (!nugetSources.length) {
+ this.buildAgent.error("Failed to fetch an enabled package source for dotnet.");
+ return [];
+ }
+ const sources = [];
+ for (const nugetSource of nugetSources) {
+ const nugetIndex = await fetch(nugetSource);
+ if (!nugetIndex?.ok) {
+ this.buildAgent.warn(`Failed to fetch data from NuGet source ${nugetSource}.`);
+ continue;
+ }
+ const resources = (await nugetIndex.json())?.resources;
+ const serviceUrl = resources?.find((s) => s["@type"].startsWith(NugetServiceType.SearchQueryService))?.["@id"];
+ if (!serviceUrl) {
+ this.buildAgent.warn(`Could not find a ${NugetServiceType.SearchQueryService} in NuGet source ${nugetSource}`);
+ continue;
+ }
+ sources.push(serviceUrl);
+ }
+ return sources;
+ }
+ async queryVersionsFromNugetSource(serviceUrl, toolName, includePrerelease) {
  const toolNameParam = encodeURIComponent(toolName.toLowerCase());
  const prereleaseParam = includePrerelease ? "true" : "false";
- const downloadPath = `${DotnetTool.nugetRoot}?q=${toolNameParam}&prerelease=${prereleaseParam}&semVerLevel=2.0.0&take=1`;
+ const downloadPath = `${serviceUrl}?q=${toolNameParam}&prerelease=${prereleaseParam}&semVerLevel=2.0.0&take=1`;
  const response = await fetch(downloadPath);
  if (!response || !response.ok) {
- this.buildAgent.info(`failed to query latest version for ${toolName} from ${downloadPath}. Status code: ${response ? response.status : "unknown"}`);
- return null;
+ this.buildAgent.warn(`failed to query latest version for ${toolName} from ${downloadPath}. Status code: ${response ? response.status : "unknown"}`);
+ return [];
  }
  const { data } = await response.json();
  const versions = data[0].versions.map((x) => x.version);
- if (!versions || !versions.length) {
+ return versions ?? [];
+ }
+ async queryLatestMatch(toolName, versionSpec, includePrerelease) {
+ this.buildAgent.info(
+ `Querying tool versions for ${toolName}${versionSpec ? `@${versionSpec}` : ""} ${includePrerelease ? "including pre-releases" : ""}`
+ );
+ const queryServices = await this.getQueryServices();
+ if (!queryServices.length) {
  return null;
  }
+ let versions = (await Promise.all(
+ queryServices.map(async (service) => await this.queryVersionsFromNugetSource(service, toolName, includePrerelease))
+ )).flat();
+ versions = [...new Set(versions)];
  this.buildAgent.debug(`got versions: ${versions.join(", ")}`);
  const version = semverExports.maxSatisfying(versions, versionSpec, { includePrerelease });
  if (version) {

dist/tools/libs/tools.mjs.map

@@ -6,7 +6,7 @@ import * as path from 'node:path'
 import * as semver from 'semver'
 import { type IBuildAgent, type ExecResult } from '@agents/common'
 import { ISettingsProvider } from './settings'
-import { NugetVersions } from './models'
+import { NugetServiceIndex, NugetServiceType, NugetVersions } from './models'
 import { ArgumentsBuilder } from './arguments-builder'
 
 export interface IDotnetTool {
@@ -18,8 +18,6 @@ export interface IDotnetTool {
 }
 
 export abstract class DotnetTool implements IDotnetTool {
- private static readonly nugetRoot: string = 'https://azuresearch-usnc.nuget.org/query'
-
  constructor(protected buildAgent: IBuildAgent) {}
 
  abstract get packageName(): string
@@ -228,29 +226,74 @@ export abstract class DotnetTool implements IDotnetTool {
  return path.normalize(workDir)
  }
 
- private async queryLatestMatch(toolName: string, versionSpec: string, includePrerelease: boolean): Promise<string | null> {
- this.buildAgent.info(
- `Querying tool versions for ${toolName}${versionSpec ? `@${versionSpec}` : ''} ${includePrerelease ? 'including pre-releases' : ''}`
- )
+ private async getQueryServices(): Promise<string[]> {
+ // Use dotnet tool to get the first enabled nuget source.
+ const builder = new ArgumentsBuilder().addArgument('nuget').addArgument('list').addArgument('source').addKeyValue('format', 'short')
+ const result = await this.execute('dotnet', builder.build())
+
+ // Each line of the output starts with either E (enabled) or D (disabled), followed by a space and index url.
+ const nugetSources = [...(result.stdout ?? '').matchAll(/^E (?<index>.+)/gm)].map(m => m.groups!.index)
 
+ if (!nugetSources.length) {
+ this.buildAgent.error('Failed to fetch an enabled package source for dotnet.')
+ return []
+ }
+
+ const sources: string[] = []
+ for (const nugetSource of nugetSources) {
+ // Fetch the nuget source index to obtain the query service
+ const nugetIndex = await fetch(nugetSource)
+ if (!nugetIndex?.ok) {
+ this.buildAgent.warn(`Failed to fetch data from NuGet source ${nugetSource}.`)
+ continue
+ }
+
+ // Parse the nuget service index and get the (first / primary) query service
+ const resources = ((await nugetIndex.json()) as NugetServiceIndex)?.resources
+ const serviceUrl = resources?.find(s => s['@type'].startsWith(NugetServiceType.SearchQueryService))?.['@id']
+
+ if (!serviceUrl) {
+ this.buildAgent.warn(`Could not find a ${NugetServiceType.SearchQueryService} in NuGet source ${nugetSource}`)
+ continue
+ }
+ sources.push(serviceUrl)
+ }
+ return sources
+ }
+
+ private async queryVersionsFromNugetSource(serviceUrl: string, toolName: string, includePrerelease: boolean): Promise<string[]> {
  const toolNameParam = encodeURIComponent(toolName.toLowerCase())
  const prereleaseParam = includePrerelease ? 'true' : 'false'
- const downloadPath = `${DotnetTool.nugetRoot}?q=${toolNameParam}&prerelease=${prereleaseParam}&semVerLevel=2.0.0&take=1`
+ const downloadPath = `${serviceUrl}?q=${toolNameParam}&prerelease=${prereleaseParam}&semVerLevel=2.0.0&take=1`
 
  const response = await fetch(downloadPath)
 
  if (!response || !response.ok) {
- this.buildAgent.info(`failed to query latest version for ${toolName} from ${downloadPath}. Status code: ${response ? response.status : 'unknown'}`)
- return null
+ this.buildAgent.warn(`failed to query latest version for ${toolName} from ${downloadPath}. Status code: ${response ? response.status : 'unknown'}`)
+ return []
  }
-
  const { data } = (await response.json()) as NugetVersions
 
  const versions = data[0].versions.map(x => x.version)
- if (!versions || !versions.length) {
+
+ return versions ?? []
+ }
+
+ private async queryLatestMatch(toolName: string, versionSpec: string, includePrerelease: boolean): Promise<string | null> {
+ this.buildAgent.info(
+ `Querying tool versions for ${toolName}${versionSpec ? `@${versionSpec}` : ''} ${includePrerelease ? 'including pre-releases' : ''}`
+ )
+
+ const queryServices = await this.getQueryServices()
+ if (!queryServices.length) {
  return null
  }
 
+ let versions = (
+ await Promise.all(queryServices.map(async service => await this.queryVersionsFromNugetSource(service, toolName, includePrerelease)))
+ ).flat()
+ versions = [...new Set(versions)] // remove duplicates
+
  this.buildAgent.debug(`got versions: ${versions.join(', ')}`)
 
  const version = semver.maxSatisfying(versions, versionSpec, { includePrerelease })

src/tools/common/dotnet-tool.ts

@@ -12,3 +12,28 @@ export type IRunner = {
 }
 
 export type NugetVersions = { data: { versions: { version: string }[] }[] }
+
+/** See the {@link https://learn.microsoft.com/en-us/nuget/api/service-index|NuGet Server API spec}*/
+export type NugetServiceIndex = {
+ version: string
+ resources: {
+ '@type': `${NugetServiceType}${`/${string}` | ''}`
+ '@id': string
+ comment: string | undefined
+ }[]
+}
+
+export enum NugetServiceType {
+ Catalog = 'Catalog',
+ PackageBaseAddress = 'PackageBaseAddress',
+ PackageDetailsUriTemplate = 'PackageDetailsUriTemplate',
+ PackagePublish = 'PackagePublish',
+ ReadmeUriTemplate = 'ReadmeUriTemplate',
+ RegistrationsBaseUrl = 'RegistrationsBaseUrl',
+ ReportAbuseUriTemplate = 'ReportAbuseUriTemplate',
+ RepositorySignatures = 'RepositorySignatures',
+ SearchAutocompleteService = 'SearchAutocompleteService',
+ SearchQueryService = 'SearchQueryService',
+ SymbolPackagePublish = 'SymbolPackagePublish',
+ VulnerabilityInfo = 'VulnerabilityInfo'
+}