Use service variants in system tests

cmd/testrunner.go

@@ -68,6 +68,7 @@ func setupTestCommand() *cobraext.Command {
 cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription)
 cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription)
 cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription)
+cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription)
 
 for testType, runner := range testrunner.TestRunners() {
 action := testTypeCommandActionFactory(runner)
@@ -176,6 +177,8 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 return cobraext.FlagParsingError(err, cobraext.DeferCleanupFlagName)
 }
 
+variantFlag, _ := cmd.Flags().GetString(cobraext.VariantFlagName)
+
 esClient, err := elasticsearch.Client()
 if err != nil {
 return errors.Wrap(err, "can't create Elasticsearch client")
@@ -189,6 +192,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 GenerateTestResult: generateTestResult,
 ESClient: esClient,
 DeferCleanup: deferCleanup,
+ServiceVariant: variantFlag,
 })
 
 results = append(results, r...)

internal/testrunner/runners/system/runner.go

@@ -9,6 +9,7 @@ import (
 "fmt"
 "io/ioutil"
 "math/rand"
+"os"
 "path/filepath"
 "strings"
 "time"
@@ -143,45 +144,83 @@ func (r *runner) run() (results []testrunner.TestResult, err error) {
 return result.WithError(errors.Wrap(err, "reading service logs directory failed"))
 }
 
-files, err := listConfigFiles(r.options.TestFolder.Path)
+dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path)
 if err != nil {
-return result.WithError(errors.Wrap(err, "failed listing test case config files"))
-}
-for _, cfgFile := range files {
-var ctxt servicedeployer.ServiceContext
-ctxt.Name = r.options.TestFolder.Package
-ctxt.Logs.Folder.Local = locationManager.ServiceLogDir()
-ctxt.Logs.Folder.Agent = ServiceLogsAgentDir
-ctxt.Test.RunID = createTestRunID()
-testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt)
-if err != nil {
-return result.WithError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile))
-}
+return result.WithError(errors.Wrap(err, "locating data stream root failed"))
+}
+if !found {
+return result.WithError(errors.New("data stream root not found"))
+}
 
-var partial []testrunner.TestResult
-if testConfig.Skip == nil {
-logger.Debugf("running test with configuration '%s'", testConfig.Name())
-partial, err = r.runTest(testConfig, ctxt)
-} else {
-logger.Warnf("skipping %s test for %s/%s: %s (details: %s)",
-TestType, r.options.TestFolder.Package, r.options.TestFolder.DataStream,
-testConfig.Skip.Reason, testConfig.Skip.Link.String())
-result := r.newResult(testConfig.Name())
-partial, err = result.WithSkip(testConfig.Skip)
-}
+cfgFiles, err := listConfigFiles(r.options.TestFolder.Path)
+if err != nil {
+return result.WithError(errors.Wrap(err, "failed listing test case config cfgFiles"))
+}
 
-results = append(results, partial...)
-tdErr := r.tearDownTest()
-if err != nil {
-return results, err
-}
-if tdErr != nil {
-return results, errors.Wrap(tdErr, "failed to tear down runner")
+devDeployPath, err := servicedeployer.FindDevDeployPath(servicedeployer.FactoryOptions{
+PackageRootPath: r.options.PackageRootPath,
+DataStreamRootPath: dataStreamPath,
+})
+if err != nil {
+return result.WithError(errors.Wrap(err, "_dev/deploy directory not found"))
+}
+
+variantsFile, err := servicedeployer.ReadVariantsFile(devDeployPath)
+if err != nil && !errors.Is(err, os.ErrNotExist) {
+return result.WithError(errors.Wrap(err, "can't read service variant"))
+}
+
+for _, cfgFile := range cfgFiles {
+for _, variantName := range r.selectVariants(variantsFile) {
+partial, err := r.runTestPerVariant(result, locationManager, cfgFile, dataStreamPath, variantName)
+results = append(results, partial...)
+if err != nil {
+return results, err
+}
 }
 }
 return results, nil
 }
 
+func (r *runner) runTestPerVariant(result *testrunner.ResultComposer, locationManager *locations.LocationManager, cfgFile, dataStreamPath, variantName string) ([]testrunner.TestResult, error) {
+serviceOptions := servicedeployer.FactoryOptions{
+PackageRootPath: r.options.PackageRootPath,
+DataStreamRootPath: dataStreamPath,
+Variant: variantName,
+}
+
+var ctxt servicedeployer.ServiceContext
+ctxt.Name = r.options.TestFolder.Package
+ctxt.Logs.Folder.Local = locationManager.ServiceLogDir()
+ctxt.Logs.Folder.Agent = ServiceLogsAgentDir
+ctxt.Test.RunID = createTestRunID()
+testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt, variantName)
+if err != nil {
+return result.WithError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile))
+}
+
+var partial []testrunner.TestResult
+if testConfig.Skip == nil {
+logger.Debugf("running test with configuration '%s'", testConfig.Name())
+partial, err = r.runTest(testConfig, ctxt, serviceOptions)
+} else {
+logger.Warnf("skipping %s test for %s/%s: %s (details: %s)",
+TestType, r.options.TestFolder.Package, r.options.TestFolder.DataStream,
+testConfig.Skip.Reason, testConfig.Skip.Link.String())
+result := r.newResult(testConfig.Name())
+partial, err = result.WithSkip(testConfig.Skip)
+}
+
+tdErr := r.tearDownTest()
+if err != nil {
+return partial, err
+}
+if tdErr != nil {
+return partial, errors.Wrap(tdErr, "failed to tear down runner")
+}
+return partial, nil
+}
+
 func createTestRunID() string {
 return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID)
 }
@@ -223,33 +262,22 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) {
 return docs, nil
 }
 
-func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext) ([]testrunner.TestResult, error) {
+func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext, serviceOptions servicedeployer.FactoryOptions) ([]testrunner.TestResult, error) {
 result := r.newResult(config.Name())
 
 pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath)
 if err != nil {
 return result.WithError(errors.Wrap(err, "reading package manifest failed"))
 }
 
-dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path)
-if err != nil {
-return result.WithError(errors.Wrap(err, "locating data stream root failed"))
-}
-if !found {
-return result.WithError(errors.New("data stream root not found"))
-}
-
-dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile))
+dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(serviceOptions.DataStreamRootPath, packages.DataStreamManifestFile))
 if err != nil {
 return result.WithError(errors.Wrap(err, "reading data stream manifest failed"))
 }
 
 // Setup service.
 logger.Debug("setting up service...")
-serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{
-PackageRootPath: r.options.PackageRootPath,
-DataStreamRootPath: dataStreamPath,
-})
+serviceDeployer, err := servicedeployer.Factory(serviceOptions)
 if err != nil {
 return result.WithError(errors.Wrap(err, "could not create service runner"))
 }
@@ -272,7 +300,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 }
 
 // Reload test config with ctx variable substitution.
-config, err = newConfig(config.Path, ctxt)
+config, err = newConfig(config.Path, ctxt, serviceOptions.Variant)
 if err != nil {
 return result.WithError(errors.Wrap(err, "unable to reload system test case configuration"))
 }
@@ -402,10 +430,10 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 }
 
 // Validate fields in docs
-fieldsValidator, err := fields.CreateValidatorForDataStream(dataStreamPath,
+fieldsValidator, err := fields.CreateValidatorForDataStream(serviceOptions.DataStreamRootPath,
 fields.WithNumericKeywordFields(config.NumericKeywordFields))
 if err != nil {
-return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", dataStreamPath))
+return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath))
 }
 
 if err := validateFields(docs, fieldsValidator, dataStream); err != nil {
@@ -631,3 +659,18 @@ func validateFields(docs []common.MapStr, fieldsValidator *fields.Validator, dat
 
 return nil
 }
+
+func (r *runner) selectVariants(variantsFile *servicedeployer.VariantsFile) []string {
+if variantsFile == nil || variantsFile.Variants == nil {
+return []string{""} // empty variants file switches to no-variant mode
+}
+
+var variantNames []string
+for k := range variantsFile.Variants {
+if r.options.ServiceVariant != "" && r.options.ServiceVariant != k {
+continue
+}
+variantNames = append(variantNames, k)
+}
+return variantNames
+}

internal/testrunner/runners/system/servicedeployer/compose.go

@@ -20,19 +20,19 @@ import (
 // a Docker Compose file.
 type DockerComposeServiceDeployer struct {
 ymlPaths []string
-sv serviceVariant
+sv ServiceVariant
 }
 
 type dockerComposeDeployedService struct {
 ctxt ServiceContext
 
 ymlPaths []string
 project string
-sv serviceVariant
+sv ServiceVariant
 }
 
 // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer.
-func NewDockerComposeServiceDeployer(ymlPaths []string, sv serviceVariant) (*DockerComposeServiceDeployer, error) {
+func NewDockerComposeServiceDeployer(ymlPaths []string, sv ServiceVariant) (*DockerComposeServiceDeployer, error) {
 return &DockerComposeServiceDeployer{
 ymlPaths: ymlPaths,
 sv: sv,
@@ -75,7 +75,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer
 opts := compose.CommandOptions{
 Env: append(
 []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)},
-d.sv.env...),
+d.sv.Env...),
 ExtraArgs: []string{"--build", "-d"},
 }
 if err := p.Up(opts); err != nil {
@@ -149,7 +149,7 @@ func (s *dockerComposeDeployedService) TearDown() error {
 downOptions := compose.CommandOptions{
 Env: append(
 []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)},
-s.sv.env...),
+s.sv.Env...),
 
 // Remove associated volumes.
 ExtraArgs: []string{"--volumes"},

internal/testrunner/runners/system/servicedeployer/factory.go

@@ -26,7 +26,7 @@ type FactoryOptions struct {
 // Factory chooses the appropriate service runner for the given data stream, depending
 // on service configuration files defined in the package or data stream.
 func Factory(options FactoryOptions) (ServiceDeployer, error) {
-devDeployPath, err := findDevDeployPath(options)
+devDeployPath, err := FindDevDeployPath(options)
 if err != nil {
 return nil, errors.Wrapf(err, "can't find \"%s\" directory", devDeployDir)
 }
@@ -60,7 +60,8 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName)
 }
 
-func findDevDeployPath(options FactoryOptions) (string, error) {
+// FindDevDeployPath function returns a path reference to the "_dev/deploy" directory.
+func FindDevDeployPath(options FactoryOptions) (string, error) {
 dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, devDeployDir)
 _, err := os.Stat(dataStreamDevDeployPath)
 if err == nil {

internal/testrunner/runners/system/servicedeployer/variants.go

@@ -15,67 +15,82 @@ import (
 "gopkg.in/yaml.v3"
 )
 
-type variantsFile struct {
+// VariantsFile describes different variants of the service under test.
+type VariantsFile struct {
 Default string `yaml:"default"`
-Variants map[string]environment
+Variants map[string]Environment
 }
 
-type environment map[string]string
+// Environment is a key-value map storing environment variables.
+type Environment map[string]string
 
-type serviceVariant struct {
-name string
-env []string // Environment variables in format of pairs: key=value
+// ServiceVariant describes a variant of the service using Environment variables.
+type ServiceVariant struct {
+Name string
+Env []string // Environment variables in format of pairs: key=value
 }
 
-func (sv *serviceVariant) active() bool {
-return sv.name != ""
+// String method returns a string representation of the service variant.
+func (sv *ServiceVariant) String() string {
+return fmt.Sprintf("ServiceVariant{Name: %s, Env: %s}", sv.Name, strings.Join(sv.Env, ","))
 }
 
-func (sv *serviceVariant) String() string {
-return fmt.Sprintf("ServiceVariant{name: %s, env: %s}", sv.name, strings.Join(sv.env, ","))
+func (sv *ServiceVariant) active() bool {
+return sv.Name != ""
 }
 
-func useServiceVariant(devDeployPath, selected string) (serviceVariant, error) {
+// ReadVariantsFile function reads available service variants.
+func ReadVariantsFile(devDeployPath string) (*VariantsFile, error) {
 variantsYmlPath := filepath.Join(devDeployPath, "variants.yml")
 _, err := os.Stat(variantsYmlPath)
 if errors.Is(err, os.ErrNotExist) {
-return serviceVariant{}, nil // no "variants.yml" present
+return nil, os.ErrNotExist
 }
 if err != nil {
-return serviceVariant{}, errors.Wrap(err, "can't stat variants file")
+return nil, errors.Wrap(err, "can't stat variants file")
 }
 
 content, err := ioutil.ReadFile(variantsYmlPath)
 if err != nil {
-return serviceVariant{}, errors.Wrap(err, "can't read variants file")
+return nil, errors.Wrap(err, "can't read variants file")
 }
 
-var f variantsFile
+var f VariantsFile
 err = yaml.Unmarshal(content, &f)
 if err != nil {
-return serviceVariant{}, errors.Wrap(err, "can't unmarshal variants file")
+return nil, errors.Wrap(err, "can't unmarshal variants file")
+}
+return &f, nil
+}
+
+func useServiceVariant(devDeployPath, selected string) (ServiceVariant, error) {
+f, err := ReadVariantsFile(devDeployPath)
+if errors.Is(err, os.ErrNotExist) {
+return ServiceVariant{}, nil // no "variants.yml" present
+} else if err != nil {
+return ServiceVariant{}, err
 }
 
 if selected == "" {
 selected = f.Default
 }
 
 if f.Default == "" {
-return serviceVariant{}, errors.New("default variant is undefined")
+return ServiceVariant{}, errors.New("default variant is undefined")
 }
 
 env, ok := f.Variants[selected]
 if !ok {
-return serviceVariant{}, fmt.Errorf(`variant "%s" is missing`, selected)
+return ServiceVariant{}, fmt.Errorf(`variant "%s" is missing`, selected)
 }
 
-return serviceVariant{
-name: selected,
-env: asEnvVarPairs(env),
+return ServiceVariant{
+Name: selected,
+Env: asEnvVarPairs(env),
 }, nil
 }
 
-func asEnvVarPairs(env environment) []string {
+func asEnvVarPairs(env Environment) []string {
 var pairs []string
 for k, v := range env {
 pairs = append(pairs, fmt.Sprintf("%s=%s", k, v))