Skip to content

Commit f2fc756

Browse files
authored
Deployment search: Add flag to return all matches. (#664)
- By default the deployment search only makes one request, returning the number of results specified in 'size' - This only works up to 10k results, but may also already stop working before that just due to the size of the response. - A new flag now switches to using a cursor to request all matches - This works by making a request, then passing the returned cursor into the next request for the next batch of results
1 parent 0b631c8 commit f2fc756

File tree

8 files changed

+245
-12
lines changed

8 files changed

+245
-12
lines changed

cmd/deployment/search.go

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@
1818
package cmddeployment
1919

2020
import (
21+
"fmt"
2122
"github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi"
23+
"github.com/elastic/cloud-sdk-go/pkg/models"
2224
"github.com/elastic/cloud-sdk-go/pkg/util/cmdutil"
23-
"github.com/spf13/cobra"
24-
2525
"github.com/elastic/ecctl/pkg/ecctl"
26+
"github.com/spf13/cobra"
2627
)
2728

2829
const searchQueryLong = `Read more about Query DSL in https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html`
@@ -51,21 +52,57 @@ var searchCmd = &cobra.Command{
5152
return err
5253
}
5354

54-
res, err := deploymentapi.Search(deploymentapi.SearchParams{
55-
API: ecctl.Get().API,
56-
Request: &sr,
57-
})
55+
returnAllMatches, _ := cmd.Flags().GetBool("all-matches")
56+
if returnAllMatches && sr.Sort == nil {
57+
return fmt.Errorf("The query must include a sort-field when using --all-matches. Example: \"sort\": [\"id\"]")
58+
}
5859

59-
if err != nil {
60-
return err
60+
batchSize, _ := cmd.Flags().GetInt32("size")
61+
62+
var result *models.DeploymentsSearchResponse
63+
var cursor string
64+
for i := 0; i < 100; i++ {
65+
sr.Cursor = cursor
66+
if returnAllMatches {
67+
// Custom batch-size to override any size already set in the input query
68+
sr.Size = batchSize
69+
}
70+
71+
res, err := deploymentapi.Search(deploymentapi.SearchParams{
72+
API: ecctl.Get().API,
73+
Request: &sr,
74+
})
75+
76+
if err != nil {
77+
return err
78+
}
79+
80+
cursor = res.Cursor
81+
82+
if result == nil {
83+
result = res
84+
result.Cursor = "" // Hide cursor in output
85+
} else {
86+
result.Deployments = append(result.Deployments, res.Deployments...)
87+
newReturnCount := *result.ReturnCount + *res.ReturnCount
88+
result.ReturnCount = &newReturnCount
89+
result.MatchCount = newReturnCount
90+
}
91+
92+
if *res.ReturnCount == 0 || !returnAllMatches {
93+
break
94+
}
6195
}
6296

63-
return ecctl.Get().Formatter.Format("deployment/search", res)
97+
return ecctl.Get().Formatter.Format("deployment/search", result)
6498
},
6599
}
66100

67101
func init() {
68102
Command.AddCommand(searchCmd)
69103
searchCmd.Flags().StringP("file", "f", "", "JSON file that contains JSON-style domain-specific language query")
70104
searchCmd.MarkFlagRequired("file")
105+
searchCmd.Flags().BoolP("all-matches", "a", false,
106+
"Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.")
107+
searchCmd.Flags().Int32("size", 500, "Defines the size per request when using the --all-matches option.")
71108
}

cmd/deployment/search_test.go

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package cmddeployment
19+
20+
import (
21+
"github.com/elastic/cloud-sdk-go/pkg/api"
22+
"github.com/elastic/cloud-sdk-go/pkg/api/mock"
23+
"github.com/elastic/cloud-sdk-go/pkg/models"
24+
"github.com/elastic/cloud-sdk-go/pkg/util/ec"
25+
"github.com/elastic/ecctl/cmd/util/testutils"
26+
"testing"
27+
)
28+
29+
func Test_searchCmd(t *testing.T) {
30+
tests := []struct {
31+
name string
32+
args testutils.Args
33+
want testutils.Assertion
34+
}{
35+
{
36+
name: "all-matches collects deployments using multiple requests",
37+
args: testutils.Args{
38+
Cmd: searchCmd,
39+
Args: []string{
40+
"search",
41+
"-f",
42+
"testdata/search_query.json",
43+
"--all-matches",
44+
"--size",
45+
"250",
46+
},
47+
Cfg: testutils.MockCfg{
48+
OutputFormat: "json",
49+
Responses: []mock.Response{
50+
mock.New200ResponseAssertion(
51+
&mock.RequestAssertion{
52+
Header: api.DefaultWriteMockHeaders,
53+
Method: "POST",
54+
Path: "/api/v1/deployments/_search",
55+
Host: api.DefaultMockHost,
56+
Body: mock.NewStructBody(models.SearchRequest{
57+
Query: &models.QueryContainer{
58+
MatchAll: struct{}{},
59+
},
60+
Size: 250,
61+
Sort: []interface{}{"id"},
62+
}),
63+
},
64+
mock.NewStructBody(models.DeploymentsSearchResponse{
65+
Cursor: "cursor1",
66+
Deployments: []*models.DeploymentSearchResponse{
67+
{ID: ec.String("d1")},
68+
{ID: ec.String("d2")},
69+
},
70+
MatchCount: 3,
71+
MinimalMetadata: nil,
72+
ReturnCount: ec.Int32(2),
73+
}),
74+
),
75+
mock.New200ResponseAssertion(
76+
&mock.RequestAssertion{
77+
Header: api.DefaultWriteMockHeaders,
78+
Method: "POST",
79+
Path: "/api/v1/deployments/_search",
80+
Host: api.DefaultMockHost,
81+
Body: mock.NewStructBody(models.SearchRequest{
82+
Cursor: "cursor1",
83+
Query: &models.QueryContainer{
84+
MatchAll: struct{}{},
85+
},
86+
Size: 250,
87+
Sort: []interface{}{"id"},
88+
}),
89+
},
90+
mock.NewStructBody(models.DeploymentsSearchResponse{
91+
Cursor: "cursor2",
92+
Deployments: []*models.DeploymentSearchResponse{
93+
{ID: ec.String("d3")},
94+
},
95+
MatchCount: 3,
96+
MinimalMetadata: nil,
97+
ReturnCount: ec.Int32(1),
98+
}),
99+
),
100+
mock.New200ResponseAssertion(
101+
&mock.RequestAssertion{
102+
Header: api.DefaultWriteMockHeaders,
103+
Method: "POST",
104+
Path: "/api/v1/deployments/_search",
105+
Host: api.DefaultMockHost,
106+
Body: mock.NewStructBody(models.SearchRequest{
107+
Cursor: "cursor2",
108+
Query: &models.QueryContainer{
109+
MatchAll: struct{}{},
110+
},
111+
Size: 250,
112+
Sort: []interface{}{"id"},
113+
}),
114+
},
115+
mock.NewStructBody(models.DeploymentsSearchResponse{
116+
Cursor: "cursor3",
117+
Deployments: []*models.DeploymentSearchResponse{},
118+
MatchCount: 3,
119+
MinimalMetadata: nil,
120+
ReturnCount: ec.Int32(0),
121+
}),
122+
),
123+
},
124+
},
125+
},
126+
want: testutils.Assertion{
127+
Stdout: string(expectedOutput) + "\n",
128+
},
129+
},
130+
{
131+
name: "all-matches requires a query with a sort",
132+
args: testutils.Args{
133+
Cmd: searchCmd,
134+
Args: []string{
135+
"search",
136+
"-f",
137+
"testdata/search_query_no_sort.json",
138+
"--all-matches",
139+
},
140+
Cfg: testutils.MockCfg{
141+
OutputFormat: "json",
142+
Responses: []mock.Response{},
143+
},
144+
},
145+
want: testutils.Assertion{
146+
Err: "The query must include a sort-field when using --all-matches. Example: \"sort\": [\"id\"]",
147+
},
148+
},
149+
}
150+
for _, tt := range tests {
151+
t.Run(tt.name, func(t *testing.T) {
152+
testutils.RunCmdAssertion(t, tt.args, tt.want)
153+
})
154+
}
155+
}
156+
157+
var expectedOutput = `{
158+
"deployments": [
159+
{
160+
"healthy": null,
161+
"id": "d1",
162+
"name": null,
163+
"resources": null
164+
},
165+
{
166+
"healthy": null,
167+
"id": "d2",
168+
"name": null,
169+
"resources": null
170+
},
171+
{
172+
"healthy": null,
173+
"id": "d3",
174+
"name": null,
175+
"resources": null
176+
}
177+
],
178+
"match_count": 3,
179+
"minimal_metadata": null,
180+
"return_count": 3
181+
}`
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"query": {
3+
"match_all": {}
4+
},
5+
"sort": ["id"]
6+
}
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"query": {
3+
"match_all": {}
4+
}
5+
}

docs/ecctl_deployment_search.adoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@ ecctl deployment search -f <query file.json> [flags]
3030
=== Options
3131

3232
----
33+
-a, --all-matches Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.
3334
-f, --file string JSON file that contains JSON-style domain-specific language query
3435
-h, --help help for search
36+
--size int32 Defines the size per request when using the --all-matches option. (default 500)
3537
----
3638

3739
[float]

docs/ecctl_deployment_search.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@ $ ecctl deployment search -f query_string_query.json
2828
### Options
2929

3030
```
31+
-a, --all-matches Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.
3132
-f, --file string JSON file that contains JSON-style domain-specific language query
3233
-h, --help help for search
34+
--size int32 Defines the size per request when using the --all-matches option. (default 500)
3335
```
3436

3537
### Options inherited from parent commands

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.20
55
require (
66
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
77
github.com/blang/semver/v4 v4.0.0
8-
github.com/elastic/cloud-sdk-go v1.20.0
8+
github.com/elastic/cloud-sdk-go v1.22.0
99
github.com/go-openapi/runtime v0.23.0
1010
github.com/go-openapi/strfmt v0.21.2
1111
github.com/pkg/errors v0.9.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
2323
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2424
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
2525
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
26-
github.com/elastic/cloud-sdk-go v1.20.0 h1:OkGG0CRXSZbntNwoKATbqO8SQoaBZAfAcXavdi8sA5Y=
27-
github.com/elastic/cloud-sdk-go v1.20.0/go.mod h1:k0ZebhZKX22l6Ysl5Zbpc8VLF54hfwDtHppEEEVUJ04=
26+
github.com/elastic/cloud-sdk-go v1.22.0 h1:sPjvu7zZeDbgl6eufy41VH0TjWbaMgDS+Cy9qIvdFZ4=
27+
github.com/elastic/cloud-sdk-go v1.22.0/go.mod h1:k0ZebhZKX22l6Ysl5Zbpc8VLF54hfwDtHppEEEVUJ04=
2828
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
2929
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
3030
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=

0 commit comments

Comments
 (0)