Skip to content

Commit 5944ba8

Browse files
P1llusP1llus
authored
[HTTPJSON] fixing config fields in manifest.yml not working as intended (#2815)
* fixing config fields in manifest.yml * ensure oauth2 is not added when not used * updating some tests * making sure processors are templated the same way as the rest of the packages * fixing changelog and changing manifest descriptions
1 parent a7f3e23 commit 5944ba8

File tree

6 files changed

+236
-90
lines changed

6 files changed

+236
-90
lines changed

‎packages/httpjson/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
- version: "1.1.0"
2+
changes:
3+
- description: Fixes issues with certain configuration fields not working
4+
type: bugfix
5+
link: https://github.com/elastic/integrations/pull/2815
16
- version: "1.0.0"
27
changes:
38
- description: Initial Implementation

‎packages/httpjson/data_stream/generic/_dev/test/system/test-transforms-config.yml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ data_stream:
77
password: test
88
request_url: http://{{Hostname}}:{{Port}}/testtransforms/api
99
request_method: POST
10-
request_body:
10+
request_body: |-
1111
message:
1212
limit: 1000
1313
request_transforms: |-
@@ -18,6 +18,3 @@ data_stream:
1818
- set:
1919
target: body.test
2020
value: success
21-
request_custom: |-
22-
encode_as: application/json
23-
timeout: 25s

‎packages/httpjson/data_stream/generic/agent/stream/httpjson.yml.hbs‎

Lines changed: 79 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,47 +14,106 @@ auth.basic.password: {{password}}
1414
pipeline: {{pipeline}}
1515
{{/if}}
1616
{{#if oauth_id}}
17-
auth.oauth2:
1817
{{#if oauth_id}}
19-
client.id: {{oauth_id}}
18+
auth.oauth2.client.id: {{oauth_id}}
2019
{{/if}}
2120
{{#if oauth_secret}}
22-
client.secret: {{oauth_secret}}
21+
auth.oauth2.client.secret: {{oauth_secret}}
2322
{{/if}}
2423
{{#if oauth_token_url}}
25-
token_url: {{oauth_token_url}}
24+
auth.oauth2.token_url: {{oauth_token_url}}
2625
{{/if}}
27-
{{#if oauth_custom}}
28-
{{oauth_custom}}
26+
{{#if oauth_provider}}
27+
auth.oauth2.provider: {{oauth_provider}}
28+
{{/if}}
29+
{{#if oauth_scopes}}
30+
auth.oauth2.scopes:
31+
{{#each oauth_scopes as |scope i|}}
32+
- {{scope}}
33+
{{/each}}
34+
{{/if}}
35+
{{#if oauth_google_credentials_file}}
36+
auth.oauth2.google.credentials_file: {{oauth_google_credentials_file}}
37+
{{/if}}
38+
{{#if oauth_google_credentials_json}}
39+
auth.oauth2.google.credentials_json: '{{oauth_google_credentials_json}}'
40+
{{/if}}
41+
{{#if oauth_google_jwt_file}}
42+
auth.oauth2.google.jwt_file: {{oauth_google_jwt_file}}
43+
{{/if}}
44+
{{#if oauth_azure_tenant_id}}
45+
auth.oauth2.azure.tenant_id: {{oauth_azure_tenant_id}}
46+
{{/if}}
47+
{{#if oauth_azure_resource}}
48+
auth.oauth2.azure.resource: {{oauth_azure_resource}}
49+
{{/if}}
50+
{{#if oauth_endpoint_params}}
51+
auth.oauth2.azure.resource:
52+
{{oauth_endpoint_params}}
2953
{{/if}}
3054
{{/if}}
3155

32-
request:
33-
url: {{request_url}}
34-
method: {{request_method}}
56+
request.url: {{request_url}}
57+
request.method: {{request_method}}
3558
{{#if request_body}}
36-
body:
37-
{{request_body}}
59+
request.body:
60+
{{request_body}}
3861
{{/if}}
3962
{{#if request_transforms}}
40-
transforms:
41-
{{request_transforms}}
63+
request.transforms:
64+
{{request_transforms}}
4265
{{/if}}
4366
{{#if request_ssl}}
44-
ssl:
45-
{{ssl}}
67+
request.ssl:
68+
{{request_ssl}}
69+
{{/if}}
70+
{{#if request_encode_as}}
71+
request.encode_as: {{request_encode_as}}
72+
{{/if}}
73+
{{#if request_timeout}}
74+
request.timeout: {{request_timeout}}
75+
{{/if}}
76+
{{#if request_proxy_url}}
77+
request.proxy_url: {{request_proxy_url}}
78+
{{/if}}
79+
{{#if request_retry_max_attempts}}
80+
request.retry.max_attempts: {{request_retry_max_attempts}}
81+
{{/if}}
82+
{{#if request_retry_wait_min}}
83+
request.retry.wait_min: {{request_retry_wait_min}}
84+
{{/if}}
85+
{{#if request_retry_wait_max}}
86+
request.retry.wait_max: {{request_retry_wait_max}}
87+
{{/if}}
88+
{{#if request_redirect_forward_headers}}
89+
request.redirect.forward_headers: {{request_redirect_forward_headers}}
90+
{{/if}}
91+
{{#if request_redirect_headers_ban_list}}
92+
request.redirect.headers_ban_list:
93+
{{#each request_redirect_headers_ban_list as |item i|}}
94+
- {{item}}
95+
{{/each}}
96+
{{/if}}
97+
{{#if request_redirect_max_redirects}}
98+
request.redirect.max_redirects: {{request_redirect_max_redirects}}
99+
{{/if}}
100+
{{#if request_rate_limit_limit}}
101+
request.rate_limit.limit: {{request_rate_limit_limit}}
102+
{{/if}}
103+
{{#if request_rate_limit_reset}}
104+
request.rate_limit.reset: {{request_rate_limit_reset}}
46105
{{/if}}
47-
{{#if request_custom}}
48-
{{request_custom}}
106+
{{#if request_rate_limit_remaining}}
107+
request.rate_limit.remaining: {{request_rate_limit_remaining}}
49108
{{/if}}
50109

51110
{{#if response_transforms}}
52111
response.transforms:
53-
{{response_transforms}}
112+
{{response_transforms}}
54113
{{/if}}
55114
{{#if response_split}}
56115
response.split:
57-
{{response_split}}
116+
{{response_split}}
58117
{{/if}}
59118
{{#if response_pagination}}
60119
response.pagination: {{response_pagination}}
@@ -82,5 +141,5 @@ publisher_pipeline.disable_host: true
82141
{{/contains}}
83142
{{#if processors}}
84143
processors:
85-
{{processors}}
144+
{{processors}}
86145
{{/if}}

‎packages/httpjson/data_stream/generic/manifest.yml‎

Lines changed: 136 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ streams:
138138
title: Custom request cursor
139139
description: |
140140
A cursor is used to keep state between each API request, and can be set to for example the value of something in the response body.
141-
More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#cursor)
141+
More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#cursor).
142142
show_user: true
143143
multi: false
144144
required: false
@@ -154,47 +154,152 @@ streams:
154154
show_user: false
155155
default: |
156156
#verification_mode: none
157-
- name: request_custom
157+
- name: request_encode_as
158+
type: text
159+
title: Request Encode As
160+
description: ContentType used for encoding the request body. If set it will force the encoding in the specified format regardless of the Content-Type header value.
161+
show_user: false
162+
multi: false
163+
required: false
164+
- name: request_timeout
165+
type: text
166+
title: Request Timeout
167+
description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. Default is "30"s.
168+
show_user: false
169+
multi: false
170+
required: false
171+
- name: request_proxy_url
172+
type: text
173+
title: Request Proxy
174+
description: This specifies proxy configuration in the form of `http[s]://<user>:<password>@<server name/ip>:<port>`.
175+
show_user: false
176+
multi: false
177+
required: false
178+
- name: request_retry_max_attempts
179+
type: text
180+
title: Request Retry Max Attempts
181+
description: The maximum number of retries for the HTTP client. Default is "5".
182+
show_user: false
183+
multi: false
184+
required: false
185+
- name: request_retry_wait_min
186+
type: text
187+
title: Request Retry Wait Min
188+
description: The minimum time to wait before a retry is attempted. Default is "1s".
189+
show_user: false
190+
multi: false
191+
required: false
192+
- name: request_retry_wait_max
193+
type: text
194+
title: Request Retry Wait Max
195+
description: The maximum time to wait before a retry is attempted. Default is "60s".
196+
show_user: false
197+
multi: false
198+
required: false
199+
- name: request_redirect_forward_headers
200+
type: bool
201+
title: Request Redirect Forward Headers
202+
description: When set to true request headers are forwarded in case of a redirect. Default is "false".
203+
show_user: false
204+
multi: false
205+
required: false
206+
- name: request_redirect_headers_ban_list
207+
type: text
208+
title: Request Redirect Headers Ban List
209+
description: When Redirect Forward Headers is set to true, all headers except the ones defined in this list will be forwarded. All headers are forwarded by default.
210+
show_user: false
211+
multi: true
212+
required: false
213+
- name: request_redirect_max_redirects
214+
type: text
215+
title: Request Redirect Max Redirects
216+
description: The maximum number of redirects to follow for a request. Default is "10".
217+
show_user: false
218+
multi: false
219+
required: false
220+
- name: request_rate_limit_limit
221+
type: text
222+
title: Request Rate Limit
223+
description: The value of the response that specifies the total limit. It is defined with a Go template value.
224+
show_user: false
225+
multi: false
226+
required: false
227+
- name: request_rate_limit_reset
228+
type: text
229+
title: Request Rate Limit Reset
230+
description: The value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value.
231+
show_user: false
232+
multi: false
233+
required: false
234+
- name: request_rate_limit_remaining
235+
type: text
236+
title: Request Rate Limit Remaining
237+
description: The value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value.
238+
show_user: false
239+
multi: false
240+
required: false
241+
- name: oauth_provider
242+
type: text
243+
title: Oauth2 Provider
244+
description: Used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. Supported providers are "azure" and "google".
245+
show_user: false
246+
multi: false
247+
required: false
248+
- name: oauth_scopes
158249
type: yaml
159-
title: Request Custom settings
160-
description: Optional Requests settings, comment out the ones to override, more information found in the httpjson [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_url)
250+
title: Oauth2 Scopes
251+
description: A list of scopes that will be requested during the oauth2 flow. It is optional for all providers.
252+
show_user: false
253+
multi: true
254+
required: false
255+
- name: oauth_google_credentials_file
256+
type: text
257+
title: Oauth2 Google Credentials File
258+
description: The full path to the credentials file for Google.
161259
show_user: false
162260
multi: false
163261
required: false
164-
default: |
165-
#encode_as: application/json
166-
#timeout: 30s
167-
#proxy_url: http[s]://<user>:<password>@<server name/ip>:<port>
168-
#retry.max_attempts: 5
169-
#retry.wait_min: 1s
170-
#retry.wait_max: 60s
171-
#redirect.forward_headers: false
172-
#redirect.headers_ban_list: []
173-
#redirect.max_redirects: 10
174-
#rate_limit.limit: '[[ .last_response.header.Get "X-RateLimit-Limit" ]]'
175-
#rate_limit.reset: '[[ .last_response.header.Get "X-RateLimit-Reset" ]]'
176-
#rate_limit.remaining: '[[ .last_response.header.Get "X-RateLimit-Remaining" ]]'
177-
- name: oauth_custom
262+
- name: oauth_google_credentials_json
263+
type: text
264+
title: Oauth2 Google Credentials JSON
265+
description: Your Google credentials information as raw JSON.
266+
show_user: false
267+
multi: false
268+
required: false
269+
- name: oauth_google_jwt_file
270+
type: text
271+
title: Oauth2 Google JWT File
272+
description: Full path to the JWT Account Key file for Google.
273+
show_user: false
274+
multi: false
275+
required: false
276+
- name: oauth_azure_tenant_id
277+
type: text
278+
title: Oauth2 Azure Tenant ID
279+
description: Optional setting used for authentication when using Azure provider. Since it is used in the process to generate the token_url, it can’t be used in combination with it.
280+
show_user: false
281+
multi: false
282+
required: false
283+
- name: oauth_azure_resource
284+
type: text
285+
title: Oauth2 Azure Resource
286+
description: Optional setting for the accessed WebAPI resource when using azure provider.
287+
show_user: false
288+
multi: false
289+
required: false
290+
- name: oauth_endpoint_params
178291
type: yaml
179-
title: Oauth2 Custom settings
180-
description: Optional Oauth2 settings, comment out the ones to override, more information found in the httpjson [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_auth_oauth2_enabled)
292+
title: Oauth2 Endpoint Params
293+
description: Set of values that will be sent on each request to the token_url. Each param key can have multiple values. Can be set for all providers except google.
181294
show_user: false
182295
multi: false
183296
required: false
184297
default: |
185-
#provider: default
186-
#scopes: []
187-
#google.credentials_file: /path/to/file
188-
#google.credentials_json: '{\"example\":\"credentials\"}'
189-
#google.jwt_file: /path/to/jwt
190-
#azure.tenant_id: ID
191-
#azure.resource: https://api.windows.com/
192-
#endpoint_params:
193-
# Param1:
298+
#Param1:
194299
# - ValueA
195300
# - ValueB
196-
# Param2:
197-
# - Value
301+
#Param2:
302+
# - Value
198303
- name: response_decode_as
199304
type: text
200305
title: Response decode settings

0 commit comments

Comments
 (0)