H2 settings (awslabs#648)

Author: TingDaoK

awscrt/http.py

@@ -1,14 +1,16 @@
 [build-system]
 requires = [
  "setuptools>=75.3.1",
- "wheel>=0.45.1", # used by our setup.py
+ "wheel>=0.45.1",  # used by our setup.py
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "awscrt"
-license = {text = "Apache-2.0", files = ["LICENSE"]}
-authors = [{name = "Amazon Web Services, Inc", email = "aws-sdk-common-runtime@amazon.com"}]
+license = { text = "Apache-2.0", files = ["LICENSE"] }
+authors = [
+ { name = "Amazon Web Services, Inc", email = "aws-sdk-common-runtime@amazon.com" },
+]
 description = "A common runtime for AWS Python projects"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -19,9 +21,7 @@ classifiers = [
  "Operating System :: Unix",
  "Operating System :: MacOS",
 ]
-dynamic = [
- "version",
-]
+dynamic = ["version"]
 
 [project.urls]
 github = "https://github.com/awslabs/aws-crt-python"
@@ -31,9 +31,10 @@ releasenotes = "https://github.com/awslabs/aws-crt-python/releases"
 
 [project.optional-dependencies]
 dev = [
- "autopep8>=2.3.1", # for code formatting
- "build>=1.2.2", # for building wheels
+ "autopep8>=2.3.1",  # for code formatting
+ "build>=1.2.2",  # for building wheels
  "sphinx>=7.2.6,<7.3; python_version >= '3.9'", # for building docs
- "websockets>=13.1", # for tests
- "h2", # for tests
+ "websockets>=13.1", # for tests
+ # for tests, restrict to exact version for test_http_client/TestClientMockServer/_on_remote_settings_changed that relies on the implementation details. Also, this is no needs to update this package.
+ "h2==4.1.0",
 ]

crt/aws-c-common

@@ -138,6 +138,85 @@ static void s_on_client_connection_setup(
  PyGILState_Release(state);
 }
 
+/* Function to convert Python list of Http2Setting to C array of aws_http2_setting */
+static int s_convert_http2_settings(
+ PyObject *initial_settings_py,
+ struct aws_allocator *allocator,
+ struct aws_http2_setting *out_settings[],
+ size_t *out_size) {
+ Py_ssize_t py_list_size = PyList_Size(initial_settings_py);
+ if (py_list_size == 0) {
+ *out_size = 0;
+ *out_settings = NULL;
+ return AWS_OP_SUCCESS;
+ }
+
+ *out_settings = aws_mem_calloc(allocator, py_list_size, sizeof(struct aws_http2_setting));
+
+ for (Py_ssize_t i = 0; i < py_list_size; i++) {
+ PyObject *setting_py = PyList_GetItem(initial_settings_py, i);
+
+ /* Get id attribute */
+ enum aws_http2_settings_id id = PyObject_GetAttrAsIntEnum(setting_py, "Http2Setting", "id");
+ if (PyErr_Occurred()) {
+ goto error;
+ }
+
+ /* Get value attribute */
+ uint32_t value = PyObject_GetAttrAsUint32(setting_py, "Http2Setting", "value");
+ if (PyErr_Occurred()) {
+ goto error;
+ }
+ (*out_settings)[i].id = id;
+ (*out_settings)[i].value = value;
+ }
+
+ *out_size = (size_t)py_list_size;
+ return AWS_OP_SUCCESS;
+error:
+ *out_size = 0;
+ aws_mem_release(allocator, *out_settings);
+ *out_settings = NULL;
+ return AWS_OP_ERR;
+}
+
+static void s_http2_on_remote_settings_change(
+ struct aws_http_connection *http2_connection,
+ const struct aws_http2_setting *settings_array,
+ size_t num_settings,
+ void *user_data) {
+ (void)http2_connection;
+ struct http_connection_binding *connection = user_data;
+
+ PyGILState_STATE state;
+ if (aws_py_gilstate_ensure(&state)) {
+ return; /* Python has shut down. Nothing matters anymore, but don't crash */
+ }
+ // Create a new list to hold tuples
+ PyObject *py_settings_list = PyList_New(num_settings);
+ if (!py_settings_list) {
+ PyErr_WriteUnraisable(PyErr_Occurred());
+ return;
+ }
+ for (size_t i = 0; i < num_settings; i++) {
+ PyObject *tuple = Py_BuildValue("(iI)", settings_array[i].id, settings_array[i].value);
+ if (!tuple) {
+ PyErr_WriteUnraisable(PyErr_Occurred());
+ goto done;
+ }
+ PyList_SetItem(py_settings_list, i, tuple); /* steals reference to tuple */
+ }
+ PyObject *result = PyObject_CallMethod(connection->py_core, "_on_remote_settings_changed", "(O)", py_settings_list);
+ if (!result) {
+ PyErr_WriteUnraisable(PyErr_Occurred());
+ goto done;
+ }
+ Py_DECREF(result);
+done:
+ Py_XDECREF(py_settings_list);
+ PyGILState_Release(state);
+}
+
 PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
  (void)self;
 
@@ -150,18 +229,23 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
  PyObject *socket_options_py;
  PyObject *tls_options_py;
  PyObject *proxy_options_py;
+ PyObject *initial_settings_py;
+ PyObject *on_remote_settings_changed_py;
  PyObject *py_core;
+ bool success = false;
 
  if (!PyArg_ParseTuple(
  args,
- "Os#IOOOO",
+ "Os#IOOOOOO",
  &bootstrap_py,
  &host_name,
  &host_name_len,
  &port_number,
  &socket_options_py,
  &tls_options_py,
  &proxy_options_py,
+ &initial_settings_py,
+ &on_remote_settings_changed_py,
  &py_core)) {
  return NULL;
  }
@@ -172,23 +256,34 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
  }
 
  struct http_connection_binding *connection = aws_mem_calloc(allocator, 1, sizeof(struct http_connection_binding));
- if (!connection) {
- return PyErr_AwsLastError();
- }
-
  /* From hereon, we need to clean up if errors occur */
+ struct aws_http2_setting *http2_settings = NULL;
+ size_t http2_settings_count = 0;
+ struct aws_http2_connection_options http2_options = {0};
 
  struct aws_tls_connection_options *tls_options = NULL;
  if (tls_options_py != Py_None) {
  tls_options = aws_py_get_tls_connection_options(tls_options_py);
  if (!tls_options) {
- goto error;
+ goto done;
  }
  }
 
  struct aws_socket_options socket_options;
  if (!aws_py_socket_options_init(&socket_options, socket_options_py)) {
- goto error;
+ goto done;
+ }
+
+ if (initial_settings_py != Py_None) {
+ /* Get the array from the pylist */
+ if (s_convert_http2_settings(initial_settings_py, allocator, &http2_settings, &http2_settings_count)) {
+ goto done;
+ }
+ http2_options.initial_settings_array = http2_settings;
+ http2_options.num_initial_settings = http2_settings_count;
+ }
+ if (on_remote_settings_changed_py != Py_None) {
+ http2_options.on_remote_settings_change = s_http2_on_remote_settings_change;
  }
 
  /* proxy options are optional */
@@ -197,7 +292,7 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
  if (proxy_options_py != Py_None) {
  proxy_options = &proxy_options_storage;
  if (!aws_py_http_proxy_options_init(proxy_options, proxy_options_py)) {
- goto error;
+ goto done;
  }
  }
 
@@ -214,21 +309,27 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
  .socket_options = &socket_options,
  .on_setup = s_on_client_connection_setup,
  .on_shutdown = s_on_connection_shutdown,
+ .http2_options = &http2_options,
  };
 
  connection->py_core = py_core;
  Py_INCREF(connection->py_core);
 
  if (aws_http_client_connect(&http_options)) {
  PyErr_SetAwsLastError();
- goto error;
+ goto done;
  }
+ success = true;
 
+done:
+ if (http2_settings) {
+ aws_mem_release(allocator, http2_settings);
+ }
+ if (!success) {
+ s_connection_destroy(connection);
+ return NULL;
+ }
  Py_RETURN_NONE;
-
-error:
- s_connection_destroy(connection);
- return NULL;
 }
 
 PyObject *aws_py_http_connection_close(PyObject *self, PyObject *args) {

crt/aws-c-http

@@ -1,22 +1,22 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-import awscrt.exceptions
-from awscrt.http import HttpClientConnection, HttpClientStream, HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion, Http2ClientConnection
-from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions, TlsCipherPref
-from concurrent.futures import Future, thread
-from http.server import HTTPServer, SimpleHTTPRequestHandler
-from io import BytesIO
-import os
-import ssl
-from test import NativeResourceTest
-import threading
-import unittest
-from urllib.parse import urlparse
-import subprocess
-import sys
-import socket
 import time
+import socket
+import sys
+import subprocess
+from urllib.parse import urlparse
+import unittest
+import threading
+from test import NativeResourceTest
+import ssl
+import os
+from io import BytesIO
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from concurrent.futures import Future, thread
+from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions, TlsCipherPref
+from awscrt.http import HttpClientConnection, HttpClientStream, HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion, Http2ClientConnection, Http2Setting, Http2SettingID
+import awscrt.exceptions
 
 
 class Response:
@@ -449,7 +449,24 @@ def tearDown(self):
  self.p_server.kill()
  super().tearDown()
 
- def _new_mock_connection(self):
+ def _on_remote_settings_changed(self, settings):
+ # The mock server has the default settings with
+ # ENABLE_PUSH = 0
+ # MAX_CONCURRENT_STREAMS = 100
+ # MAX_HEADER_LIST_SIZE = 2**16
+ # using h2@4.1.0, code can be found in
+ # https://github.com/python-hyper/h2/blob/191ac06e0949fcfe3367b06eeb101a5a1a335964/src/h2/connection.py#L340-L359
+ # Check the settings here
+ self.assertEqual(len(settings), 3)
+ for i in settings:
+ if i.id == Http2SettingID.ENABLE_PUSH:
+ self.assertEqual(i.value, 0)
+ if i.id == Http2SettingID.MAX_CONCURRENT_STREAMS:
+ self.assertEqual(i.value, 100)
+ if i.id == Http2SettingID.MAX_HEADER_LIST_SIZE:
+ self.assertEqual(i.value, 2**16)
+
+ def _new_mock_connection(self, initial_settings=None):
 
  event_loop_group = EventLoopGroup()
  host_resolver = DefaultHostResolver(event_loop_group)
@@ -465,11 +482,15 @@ def _new_mock_connection(self):
  tls_conn_opt = tls_ctx.new_connection_options()
  tls_conn_opt.set_server_name(self.mock_server_url.hostname)
  tls_conn_opt.set_alpn_list(["h2"])
+ if initial_settings is None:
+ initial_settings = [Http2Setting(Http2SettingID.ENABLE_PUSH, 0)]
 
  connection_future = Http2ClientConnection.new(host_name=self.mock_server_url.hostname,
  port=port,
  bootstrap=bootstrap,
- tls_connection_options=tls_conn_opt)
+ tls_connection_options=tls_conn_opt,
+ initial_settings=initial_settings,
+ on_remote_settings_changed=self._on_remote_settings_changed)
  return connection_future.result(self.timeout)
 
  def test_h2_mock_server_manual_write(self):
@@ -571,6 +592,37 @@ def test_h2_mock_server_manual_write_lifetime(self):
  stream.completion_future.result()
  self.assertEqual(None, connection.close().exception(self.timeout))
 
+ def test_h2_mock_server_settings(self):
+ exception = None
+ try:
+ # invalid settings, should throw an exception
+ initial_settings = [100]
+ connection = self._new_mock_connection(initial_settings)
+ except Exception as e:
+ exception = e
+ self.assertIsNotNone(exception)
+
+ connection = self._new_mock_connection()
+ # check we set an h2 connection
+ self.assertEqual(connection.version, HttpVersion.Http2)
+
+ request = HttpRequest('POST', self.mock_server_url.path)
+ request.headers.add('host', self.mock_server_url.hostname)
+ response = Response()
+ stream = connection.request(request, response.on_response, response.on_body, manual_write=True)
+ stream.activate()
+ exception = None
+ stream.write_data(BytesIO(b'hello'), True)
+
+ self.assertIsNone(exception)
+ stream_completion_result = stream.completion_future.result(80)
+ # check result
+ self.assertEqual(200, response.status_code)
+ self.assertEqual(200, stream_completion_result)
+ print(response.body)
+
+ self.assertEqual(None, connection.close().exception(self.timeout))
+
 
 if __name__ == '__main__':
  unittest.main()