Merge branch 'master' into fix-hooks-oom

2 files changed, 218 insertions, 0 deletions

diff --git a/strutil/matchcounter.go b/strutil/matchcounter.go
new file mode 100644
index 0000000000..09187967b9
--- /dev/null
+++ b/strutil/matchcounter.go
@@ -0,0 +1,81 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+
+/*
+ * Copyright (C) 2018 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package strutil
+
+import (
+	"bytes"
+	"regexp"
+)
+
+// A MatchCounter is a discarding io.Writer that retains up to N
+// matches to its Regexp before just counting matches.
+//
+// It does not work with regexps that cross newlines; in fact it will
+// probably not work if the data written isn't line-orineted.
+type MatchCounter struct {
+	// Regexp to use to find matches in the stream
+	Regexp *regexp.Regexp
+	// Maximum number of matches to keep; if < 0, keep all matches
+	N int
+
+	count int
+	matches []string
+	partial []byte
+}
+
+func (w *MatchCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	if len(w.partial) > 0 {
+	idx := bytes.IndexByte(p, '\n')
+	if idx < 0 {
+	w.partial = append(w.partial, p...)
+	return n, nil
+	}
+	idx++
+	w.check(append(w.partial, p[:idx]...))
+	p = p[idx:]
+	w.partial = nil
+	}
+	idx := bytes.LastIndexByte(p, '\n')
+	if idx < 0 {
+	w.partial = p
+	return n, nil
+	}
+	idx++
+	w.partial = p[idx:]
+	w.check(p[:idx])
+	return n, nil
+}
+
+func (w *MatchCounter) check(p []byte) {
+	matches := w.Regexp.FindAll(p, -1)
+	for _, match := range matches {
+	if w.N >= 0 && len(w.matches) >= w.N {
+	break
+	}
+	w.matches = append(w.matches, string(match))
+	}
+	w.count += len(matches)
+}
+
+// Matches returns the first few matches, and the total number of matches seen.
+func (w *MatchCounter) Matches() ([]string, int) {
+	return w.matches, w.count
+}
diff --git a/strutil/matchcounter_test.go b/strutil/matchcounter_test.go
new file mode 100644
index 0000000000..6efd9e7aa9
--- /dev/null
+++ b/strutil/matchcounter_test.go
@@ -0,0 +1,137 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+
+/*
+ * Copyright (C) 2018 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package strutil_test
+
+import (
+	"regexp"
+
+	"gopkg.in/check.v1"
+
+	"github.com/snapcore/snapd/strutil"
+)
+
+type mcSuite struct{}
+
+var _ = check.Suite(&mcSuite{})
+
+const out = `
+
+Failed to write /tmp/1/modules/4.4.0-112-generic/modules.symbols, skipping
+
+Write on output file failed because No space left on device
+
+Hello I am a happy line that does not mention failure.
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/modules/4.4.0-112-generic/modules.symbols.bin, skipping
+
+Write on output file failed because No space left on device
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/modules/4.4.0-112-generic/vdso/vdso32.so, skipping
+
+Write on output file failed because No space left on device
+
+La la la.
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/modules/4.4.0-112-generic/vdso/vdso64.so, skipping
+
+Write on output file failed because No space left on device
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/modules/4.4.0-112-generic/vdso/vdsox32.so, skipping
+
+Write on output file failed because No space left on device
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/snap/manifest.yaml, skipping
+
+🦄🌈💩
+
+Write on output file failed because No space left on device
+
+writer: failed to write data block 0
+
+Failed to write /tmp/1/snap/snapcraft.yaml, skipping
+`
+
+var thisRegexp = regexp.MustCompile("(?m).*[Ff]ailed.*")
+
+func (mcSuite) TestMatchCounterFull(c *check.C) {
+	// check a single write
+	expected := thisRegexp.FindAllString(out, 3)
+	w := &strutil.MatchCounter{Regexp: thisRegexp, N: 3}
+	_, err := w.Write([]byte(out))
+	c.Assert(err, check.IsNil)
+	matches, count := w.Matches()
+	c.Check(count, check.Equals, 19)
+	c.Assert(matches, check.DeepEquals, expected)
+}
+
+func (mcSuite) TestMatchCounterPartials(c *check.C) {
+	// now we know the whole thing matches expected, we check partials
+	buf := []byte(out)
+	expected := []string{
+	"Failed to write /tmp/1/modules/4.4.0-112-generic/modules.symbols, skipping",
+	"Write on output file failed because No space left on device",
+	"writer: failed to write data block 0",
+	}
+
+	for step := 1; step < 100; step++ {
+	w := &strutil.MatchCounter{Regexp: thisRegexp, N: 3}
+	var i int
+	for i = 0; i+step < len(buf); i += step {
+	_, err := w.Write(buf[i : i+step])
+	c.Assert(err, check.IsNil, check.Commentf("step:%d i:%d", step, i))
+	}
+	_, err := w.Write(buf[i:])
+	c.Assert(err, check.IsNil, check.Commentf("step:%d tail", step))
+	matches, count := w.Matches()
+	c.Check(count, check.Equals, 19, check.Commentf("step:%d", step))
+	c.Check(matches, check.DeepEquals, expected, check.Commentf("step:%d", step))
+	}
+}
+
+func (mcSuite) TestMatchCounterZero(c *check.C) {
+	w := &strutil.MatchCounter{Regexp: thisRegexp, N: 0}
+	_, err := w.Write([]byte(out))
+	c.Assert(err, check.IsNil)
+	matches, count := w.Matches()
+	c.Check(count, check.Equals, 19)
+	c.Assert(matches, check.HasLen, 0)
+}
+
+func (mcSuite) TestMatchCounterNegative(c *check.C) {
+	expected := thisRegexp.FindAllString(out, -1)
+
+	w := &strutil.MatchCounter{Regexp: thisRegexp, N: -1}
+	_, err := w.Write([]byte(out))
+	c.Assert(err, check.IsNil)
+	matches, count := w.Matches()
+	c.Check(count, check.Equals, 19)
+	c.Check(count, check.Equals, len(matches))
+	c.Assert(matches, check.DeepEquals, expected)
+}