Merge pull request #11716 from stolowski/use-seed-parallelism

snap-bootstrap, o/devicestate: use seed parallelism

5 files changed, 91 insertions, 5 deletions

diff --git a/cmd/snap-bootstrap/initramfs_mounts_state.go b/cmd/snap-bootstrap/initramfs_mounts_state.go
index 4b4a3a0902..0b3c19d70a 100644
--- a/cmd/snap-bootstrap/initramfs_mounts_state.go
+++ b/cmd/snap-bootstrap/initramfs_mounts_state.go
@@ -24,6 +24,7 @@ import (
	"fmt"
	"os"
	"path/filepath"
+	"runtime"
 
	"github.com/snapcore/snapd/asserts"
	"github.com/snapcore/snapd/boot"
@@ -36,6 +37,7 @@ import (
 
 var (
	osutilSetTime = osutil.SetTime
+	runtimeNumCPU = runtime.NumCPU
 )
 
 // initramfsMountsState helps tracking the state and progress
@@ -70,7 +72,11 @@ func (mst *initramfsMountsState) ReadEssential(recoverySystem string, essentialT
	// the RTC does not have a battery or is otherwise unreliable, etc.
	now := timeNow()
 
-	model, snaps, newTrustedEarliestTime, err := seed.ReadSystemEssentialAndBetterEarliestTime(boot.InitramfsUbuntuSeedDir, recoverySystem, essentialTypes, now, perf)
+	jobs := 1
+	if runtimeNumCPU() > 1 {
+	jobs = 2
+	}
+	model, snaps, newTrustedEarliestTime, err := seed.ReadSystemEssentialAndBetterEarliestTime(boot.InitramfsUbuntuSeedDir, recoverySystem, essentialTypes, now, jobs, perf)
	if err != nil {
	return nil, nil, err
	}
diff --git a/overlord/devicestate/firstboot.go b/overlord/devicestate/firstboot.go
index 52e094bee4..8e1ea10cf0 100644
--- a/overlord/devicestate/firstboot.go
+++ b/overlord/devicestate/firstboot.go
@@ -22,6 +22,7 @@ package devicestate
 import (
	"errors"
	"fmt"
+	"runtime"
	"sort"
 
	"github.com/snapcore/snapd/asserts"
@@ -39,6 +40,8 @@ import (
 
 var errNothingToDo = errors.New("nothing to do")
 
+var runtimeNumCPU = runtime.NumCPU
+
 func installSeedSnap(st *state.State, sn *seed.Snap, flags snapstate.Flags) (*state.TaskSet, *snap.Info, error) {
	if sn.Required {
	flags.Required = true
@@ -395,6 +398,12 @@ var loadDeviceSeed = func(st *state.State, sysLabel string) (deviceSeed seed.See
	return nil, err
	}
 
+	if runtimeNumCPU() > 1 {
+	// XXX set parallelism experimentally to 2 as I/O
+	// itself becomes a bottleneck ultimately
+	deviceSeed.SetParallelism(2)
+	}
+
	// collect and
	// set device,model from the model assertion
	commitTo := func(batch *asserts.Batch) error {
diff --git a/seed/export_test.go b/seed/export_test.go
index 610825422b..859375d35e 100644
--- a/seed/export_test.go
+++ b/seed/export_test.go
@@ -21,6 +21,7 @@ package seed
 
 import (
	"github.com/snapcore/snapd/seed/internal"
+	"github.com/snapcore/snapd/testutil"
 )
 
 type InternalSnap16 = internal.Snap16
@@ -28,3 +29,22 @@ type InternalSnap16 = internal.Snap16
 var (
	LoadAssertions = loadAssertions
 )
+
+func MockOpen(f func(seedDir, label string) (Seed, error)) (restore func()) {
+	r := testutil.Backup(&open)
+	open = f
+	return r
+}
+
+type TestSeed20 struct {
+	*seed20
+	Jobs int
+}
+
+func NewTestSeed20(s Seed) *TestSeed20 {
+	return &TestSeed20{s.(*seed20), 0}
+}
+
+func (s *TestSeed20) SetParallelism(n int) {
+	s.Jobs = n
+}
diff --git a/seed/seed.go b/seed/seed.go
index 2313c7e789..cf6bd96fd7 100644
--- a/seed/seed.go
+++ b/seed/seed.go
@@ -34,6 +34,8 @@ import (
 var (
	ErrNoAssertions = errors.New("no seed assertions")
	ErrNoMeta = errors.New("no seed metadata")
+
+	open = Open
 )
 
 // Snap holds the details of a snap in a seed.
@@ -219,21 +221,26 @@ func ReadSystemEssential(seedDir, label string, essentialTypes []snap.Type, tm t
 // ReadSystemEssentialAndBetterEarliestTime retrieves in one go
 // information about the model and essential snaps of the given types
 // for the Core 20 recovery system seed specified by seedDir and label
-// (which cannot be empty).
+// (which cannot be empty). numJobs specifies the suggested number of
+// jobs to run in parallel (0 disables parallelism).
 // It can operate even if current system time is unreliable by taking
 // a earliestTime lower bound for current time.
 // It returns as well an improved lower bound by considering
 // appropriate assertions in the seed.
-func ReadSystemEssentialAndBetterEarliestTime(seedDir, label string, essentialTypes []snap.Type, earliestTime time.Time, tm timings.Measurer) (*asserts.Model, []*Snap, time.Time, error) {
+func ReadSystemEssentialAndBetterEarliestTime(seedDir, label string, essentialTypes []snap.Type, earliestTime time.Time, numJobs int, tm timings.Measurer) (*asserts.Model, []*Snap, time.Time, error) {
	if label == "" {
	return nil, nil, time.Time{}, fmt.Errorf("system label cannot be empty")
	}
-	seed20, err := Open(seedDir, label)
+	seed20, err := open(seedDir, label)
	if err != nil {
	return nil, nil, time.Time{}, err
 
	}
 
+	if numJobs > 0 {
+	seed20.SetParallelism(numJobs)
+	}
+
	improve := func(a asserts.Assertion) {
	// we consider only snap-revision and snap-declaration
	// assertions here as they must be store-signed, see
diff --git a/seed/seed20_test.go b/seed/seed20_test.go
index 4911ae5731..304e7f2bf6 100644
--- a/seed/seed20_test.go
+++ b/seed/seed20_test.go
@@ -1093,7 +1093,7 @@ func (s *seed20Suite) TestReadSystemEssentialAndBetterEarliestTime(c *C) {
 
	for _, t := range tests {
	// test short-cut helper as well
-	mod, essSnaps, betterTime, err := seed.ReadSystemEssentialAndBetterEarliestTime(s.SeedDir, sysLabel, t.onlyTypes, earliestTime, s.perfTimings)
+	mod, essSnaps, betterTime, err := seed.ReadSystemEssentialAndBetterEarliestTime(s.SeedDir, sysLabel, t.onlyTypes, earliestTime, 0, s.perfTimings)
	c.Assert(err, IsNil)
	c.Check(mod.BrandID(), Equals, "my-brand")
	c.Check(mod.Model(), Equals, "my-model")
@@ -1122,6 +1122,50 @@ func (s *seed20Suite) TestReadSystemEssentialAndBetterEarliestTime(c *C) {
	}
 }
 
+func (s *seed20Suite) TestReadSystemEssentialAndBetterEarliestTimeParallelism(c *C) {
+	var testSeed *seed.TestSeed20
+	restore := seed.MockOpen(func(seedDir, label string) (seed.Seed, error) {
+	sd, err := seed.Open(seedDir, label)
+	testSeed = seed.NewTestSeed20(sd)
+	return testSeed, err
+	})
+	defer restore()
+
+	r := seed.MockTrusted(s.StoreSigning.Trusted)
+	defer r()
+
+	s.makeSnap(c, "snapd", "")
+	s.makeSnap(c, "core20", "")
+	s.makeSnap(c, "pc-kernel=20", "")
+	s.makeSnap(c, "pc=20", "")
+
+	sysLabel := "20191018"
+	s.MakeSeed(c, sysLabel, "my-brand", "my-model", map[string]interface{}{
+	"display-name": "my model",
+	"timestamp": time.Now().Format(time.RFC3339),
+	"architecture": "amd64",
+	"base": "core20",
+	"snaps": []interface{}{
+	map[string]interface{}{
+	"name": "pc-kernel",
+	"id": s.AssertedSnapID("pc-kernel"),
+	"type": "kernel",
+	"default-channel": "20",
+	},
+	map[string]interface{}{
+	"name": "pc",
+	"id": s.AssertedSnapID("pc"),
+	"type": "gadget",
+	"default-channel": "20",
+	}},
+	}, nil)
+
+	_, _, _, err := seed.ReadSystemEssentialAndBetterEarliestTime(s.SeedDir, sysLabel, nil, time.Time{}, 3, s.perfTimings)
+	c.Assert(err, IsNil)
+	c.Assert(testSeed, NotNil)
+	c.Check(testSeed.Jobs, Equals, 3)
+}
+
 func (s *seed20Suite) TestLoadEssentialAndMetaCore20(c *C) {
	r := seed.MockTrusted(s.StoreSigning.Trusted)
	defer r()