[DNM] Add documentation for hypothetical pre-timeout scripts

.config/nextest.toml

@@ -62,7 +62,7 @@ max-threads = 8
 [test-groups.unused-group]
 max-threads = 8
 
-[script.build-seed-archive]
+[script.setup.build-seed-archive]
 # This command builds a seed archive that's used by the integration-tests
 # package. This archive is not currently used by the older nextest-runner
 # integration framework, but that should really go away at some point.

fixtures/nextest-tests/.config/nextest.toml

@@ -91,8 +91,8 @@ max-threads = 4
 [test-groups.unused]
 max-threads = 20
 
-[script.my-script-unix]
+[script.setup.my-script-unix]
 command = './scripts/my-script.sh'
 
-[script.my-script-windows]
+[script.setup.my-script-windows]
 command = 'cmd /c "scripts\\my-script.bat"'

nextest-runner/src/config/config_impl.rs

@@ -4,16 +4,17 @@
 use super::{
  ArchiveConfig, CompiledByProfile, CompiledData, CompiledDefaultFilter, ConfigExperimental,
  CustomTestGroup, DefaultJunitImpl, DeserializedOverride, DeserializedProfileScriptConfig,
- JunitConfig, JunitImpl, NextestVersionDeserialize, RetryPolicy, ScriptConfig, ScriptId,
- SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
+ JunitConfig, JunitImpl, NextestVersionDeserialize, RetryPolicy, ScriptId, SettingSource,
+ SetupScriptConfig, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
  TestThreads, ThreadsRequired, ToolConfigFile,
 };
 use crate::{
+ config::{PreTimeoutScript, PreTimeoutScriptConfig, ScriptType},
  errors::{
  provided_by_tool, ConfigParseError, ConfigParseErrorKind, ProfileNotFound,
  UnknownConfigScriptError, UnknownTestGroupError,
  },
- list::TestList,
+ list::{TestInstance, TestList},
  platform::BuildPlatforms,
  reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplay},
 };
@@ -217,7 +218,7 @@ impl NextestConfig {
  let mut compiled = CompiledByProfile::for_default_config();
 
  let mut known_groups = BTreeSet::new();
- let mut known_scripts = BTreeSet::new();
+ let mut known_scripts = BTreeMap::new();
 
  // Next, merge in tool configs.
  for ToolConfigFile { config_file, tool } in tool_config_files_rev {
@@ -289,7 +290,7 @@ impl NextestConfig {
  experimental: &BTreeSet<ConfigExperimental>,
  unknown_callback: &mut impl FnMut(&Utf8Path, Option<&str>, &BTreeSet<String>),
  known_groups: &mut BTreeSet<CustomTestGroup>,
- known_scripts: &mut BTreeSet<ScriptId>,
+ known_scripts: &mut BTreeMap<ScriptId, ScriptType>,
  ) -> Result<(), ConfigParseError> {
  // Try building default builder + this file to get good error attribution and handle
  // overrides additively.
@@ -328,8 +329,8 @@ impl NextestConfig {
 
  known_groups.extend(valid_groups);
 
- // If scripts are present, check that the experimental feature is enabled.
- if !this_config.scripts.is_empty()
+ // If setup scripts are present, check that the experimental feature is enabled.
+ if !this_config.scripts.setup.is_empty()
  && !experimental.contains(&ConfigExperimental::SetupScripts)
  {
  return Err(ConfigParseError::new(
@@ -341,9 +342,33 @@ impl NextestConfig {
  ));
  }
 
- // Check that setup scripts are named as expected.
- let (valid_scripts, invalid_scripts): (BTreeSet<_>, _) =
- this_config.scripts.keys().cloned().partition(|script| {
+ // If pre-timeout scripts are present, check that the experimental feature is enabled.
+ if !this_config.scripts.pre_timeout.is_empty()
+ && !experimental.contains(&ConfigExperimental::PreTimeoutScripts)
+ {
+ return Err(ConfigParseError::new(
+ config_file,
+ tool,
+ ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
+ feature: ConfigExperimental::PreTimeoutScripts,
+ },
+ ));
+ }
+
+ if let Some(dup) = this_config.scripts.duplicate_ids().next() {
+ return Err(ConfigParseError::new(
+ config_file,
+ tool,
+ ConfigParseErrorKind::DuplicateConfigScriptName(dup.clone()),
+ ));
+ }
+
+ // Check that scripts are named as expected.
+ let (valid_scripts, invalid_scripts): (BTreeSet<_>, _) = this_config
+ .scripts
+ .all_script_ids()
+ .cloned()
+ .partition(|script| {
  if let Some(tool) = tool {
  // The first component must be the tool name.
  script
@@ -365,7 +390,10 @@ impl NextestConfig {
  return Err(ConfigParseError::new(config_file, tool, kind));
  }
 
- known_scripts.extend(valid_scripts);
+ known_scripts.extend(valid_scripts.into_iter().map(|id| {
+ let script_type = this_config.scripts.script_type(&id);
+ (id, script_type)
+ }));
 
  let this_config = this_config.into_config_impl();
 
@@ -434,45 +462,76 @@ impl NextestConfig {
 
  // Check that scripts are known.
  let mut unknown_script_errors = Vec::new();
- let mut check_script_ids = |profile_name: &str, scripts: &[ScriptId]| {
- if !scripts.is_empty() && !experimental.contains(&ConfigExperimental::SetupScripts) {
- return Err(ConfigParseError::new(
- config_file,
- tool,
- ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
- feature: ConfigExperimental::SetupScripts,
- },
- ));
- }
- for script in scripts {
- if !known_scripts.contains(script) {
- unknown_script_errors.push(UnknownConfigScriptError {
- profile_name: profile_name.to_owned(),
- name: script.clone(),
- });
+ let mut check_script_ids =
+ |profile_name: &str, script_type: ScriptType, scripts: &[ScriptId]| {
+ let config_feature = match script_type {
+ ScriptType::Setup => ConfigExperimental::SetupScripts,
+ ScriptType::PreTimeout => ConfigExperimental::PreTimeoutScripts,
+ };
+ if !scripts.is_empty() && !experimental.contains(&config_feature) {
+ return Err(ConfigParseError::new(
+ config_file,
+ tool,
+ ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
+ feature: config_feature,
+ },
+ ));
+ }
+ for script in scripts {
+ match known_scripts.get(script) {
+ None => {
+ unknown_script_errors.push(UnknownConfigScriptError {
+ profile_name: profile_name.to_owned(),
+ name: script.clone(),
+ });
+ }
+ Some(actual_script_type) if *actual_script_type != script_type => {
+ return Err(ConfigParseError::new(
+ config_file,
+ tool,
+ ConfigParseErrorKind::WrongConfigScriptType {
+ script: script.clone(),
+ attempted: script_type,
+ actual: actual_script_type.clone(),
+ },
+ ));
+ }
+ Some(_) => (),
+ }
  }
- }
 
- Ok(())
- };
+  Ok(())
+  };
 
  this_compiled
  .default
  .scripts
  .iter()
- .try_for_each(|scripts| check_script_ids("default", &scripts.setup))?;
+ .try_for_each(|scripts| {
+ check_script_ids("default", ScriptType::Setup, &scripts.setup)?;
+ check_script_ids(
+ "default",
+ ScriptType::PreTimeout,
+ scripts.pre_timeout.as_slice(),
+ )
+ })?;
  this_compiled
  .other
  .iter()
  .try_for_each(|(profile_name, data)| {
- data.scripts
- .iter()
- .try_for_each(|scripts| check_script_ids(profile_name, &scripts.setup))
+ data.scripts.iter().try_for_each(|scripts| {
+ check_script_ids(profile_name, ScriptType::Setup, &scripts.setup)?;
+ check_script_ids(
+ profile_name,
+ ScriptType::PreTimeout,
+ scripts.pre_timeout.as_slice(),
+ )
+ })
  })?;
 
  // If there were any unknown scripts, error out.
  if !unknown_script_errors.is_empty() {
- let known_scripts = known_scripts.iter().cloned().collect();
+ let known_scripts = known_scripts.keys().cloned().collect();
  return Err(ConfigParseError::new(
  config_file,
  tool,
@@ -578,8 +637,7 @@ pub struct EarlyProfile<'cfg> {
  default_profile: &'cfg DefaultProfileImpl,
  custom_profile: Option<&'cfg CustomProfileImpl>,
  test_groups: &'cfg BTreeMap<CustomTestGroup, TestGroupConfig>,
- // This is ordered because the scripts are used in the order they're defined.
- scripts: &'cfg IndexMap<ScriptId, ScriptConfig>,
+ scripts: &'cfg Scripts,
  // Invariant: `compiled_data.default_filter` is always present.
  pub(super) compiled_data: CompiledData<PreBuildPlatform>,
 }
@@ -646,7 +704,7 @@ pub struct EvaluatableProfile<'cfg> {
  custom_profile: Option<&'cfg CustomProfileImpl>,
  test_groups: &'cfg BTreeMap<CustomTestGroup, TestGroupConfig>,
  // This is ordered because the scripts are used in the order they're defined.
- scripts: &'cfg IndexMap<ScriptId, ScriptConfig>,
+ scripts: &'cfg Scripts,
  // Invariant: `compiled_data.default_filter` is always present.
  pub(super) compiled_data: CompiledData<FinalConfig>,
  // The default filter that's been resolved after considering overrides (i.e.
@@ -683,8 +741,8 @@ impl<'cfg> EvaluatableProfile<'cfg> {
  }
 
  /// Returns the global script configuration.
- pub fn script_config(&self) -> &'cfg IndexMap<ScriptId, ScriptConfig> {
- self.scripts
+ pub fn script_config(&self) -> &'cfg Scripts {
+ &self.scripts
  }
 
  /// Returns the retry count for this profile.
@@ -777,6 +835,11 @@ impl<'cfg> EvaluatableProfile<'cfg> {
  SetupScripts::new(self, test_list)
  }
 
+ /// Returns the pre-timeout script for the specified test, if it exists.
+ pub fn pre_timeout_script(&self, test_instance: &TestInstance<'_>) -> Option<PreTimeoutScript> {
+ PreTimeoutScript::new(self, test_instance)
+ }
+
  /// Returns settings for individual tests.
  pub fn settings_for(&self, query: &TestQuery<'_>) -> TestSettings {
  TestSettings::new(self, query)
@@ -809,7 +872,7 @@ impl<'cfg> EvaluatableProfile<'cfg> {
 pub(super) struct NextestConfigImpl {
  store: StoreConfigImpl,
  test_groups: BTreeMap<CustomTestGroup, TestGroupConfig>,
- scripts: IndexMap<ScriptId, ScriptConfig>,
+ scripts: Scripts,
  default_profile: DefaultProfileImpl,
  other_profiles: HashMap<String, CustomProfileImpl>,
 }
@@ -863,7 +926,7 @@ struct NextestConfigDeserialize {
  #[serde(default)]
  test_groups: BTreeMap<CustomTestGroup, TestGroupConfig>,
  #[serde(default, rename = "script")]
- scripts: IndexMap<ScriptId, ScriptConfig>,
+ scripts: Scripts,
  #[serde(rename = "profile")]
  profiles: HashMap<String, CustomProfileImpl>,
 }
@@ -962,7 +1025,7 @@ impl DefaultProfileImpl {
  &self.overrides
  }
 
- pub(super) fn setup_scripts(&self) -> &[DeserializedProfileScriptConfig] {
+ pub(super) fn scripts(&self) -> &[DeserializedProfileScriptConfig] {
  &self.scripts
  }
 }
@@ -1024,6 +1087,47 @@ impl CustomProfileImpl {
  }
 }
 
+/// The scripts defined in a profile.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Scripts {
+ // These maps are ordered because scripts are used in the order they're defined.
+ /// The setup scripts defined in a profile.
+ #[serde(default)]
+ pub setup: IndexMap<ScriptId, SetupScriptConfig>,
+ /// The pre-timeout scripts defined in a profile.
+ #[serde(default)]
+ pub pre_timeout: IndexMap<ScriptId, PreTimeoutScriptConfig>,
+}
+
+impl Scripts {
+ /// Determines the type of the script with the given ID.
+ ///
+ /// Panics if the ID is invalid.
+ fn script_type(&self, id: &ScriptId) -> ScriptType {
+ if self.setup.contains_key(id) {
+ ScriptType::Setup
+ } else if self.pre_timeout.contains_key(id) {
+ ScriptType::PreTimeout
+ } else {
+ panic!("Scripts::script_type called with invalid script ID: {id}")
+ }
+ }
+
+ /// Returns an iterator over the names of all scripts of all types.
+ fn all_script_ids(&self) -> impl Iterator<Item = &ScriptId> {
+ self.setup.keys().chain(self.pre_timeout.keys())
+ }
+
+ /// Returns an iterator over names that are used by more than one type of
+ /// script.
+ fn duplicate_ids(&self) -> impl Iterator<Item = &ScriptId> {
+ self.pre_timeout
+ .keys()
+ .filter(|k| self.setup.contains_key(*k))
+ }
+}
+
 #[cfg(test)]
 mod tests {
  use super::*;

nextest-runner/src/config/nextest_version.rs

@@ -232,11 +232,13 @@ impl NextestVersionConfig {
 pub enum ConfigExperimental {
  /// Enable support for setup scripts.
  SetupScripts,
+ /// Enable support for pre-timeout scripts.
+ PreTimeoutScripts,
 }
 
 impl ConfigExperimental {
  fn known() -> impl Iterator<Item = Self> {
- vec![Self::SetupScripts].into_iter()
+ vec![Self::SetupScripts, Self::PreTimeoutScripts].into_iter()
  }
 }
 
@@ -246,6 +248,7 @@ impl FromStr for ConfigExperimental {
  fn from_str(s: &str) -> Result<Self, Self::Err> {
  match s {
  "setup-scripts" => Ok(Self::SetupScripts),
+ "pre-timeout-scripts" => Ok(Self::PreTimeoutScripts),
  _ => Err(()),
  }
  }
@@ -255,6 +258,7 @@ impl fmt::Display for ConfigExperimental {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  match self {
  Self::SetupScripts => write!(f, "setup-scripts"),
+ Self::PreTimeoutScripts => write!(f, "pre-timeout-scripts"),
  }
  }
 }