Use writev() to send responses

Massive performance improvements!

Author: lpereira

README

@@ -6,7 +6,7 @@ It is meant to be only an exercise in writing a server using BSD sockets and
 other auxilliary APIs, like epoll. It is not meant to serve things on the
 Internet, as it isn't resilient enough.
 
-This can yield up to 40000 requests/s on a dual-core Core i7 processor. Not
-much, but for something written in just over a day, it's not as bad as it
-sounds. I'm open to suggestions that can improve performance; better yet if
-you can fork it and actually make it happen.
+This can yield around 175000 requests/s on a dual-core Core i7 processor. 
+Not too shabby for something written in about 20 days. I'm open to
+suggestions that can improve performance; better yet if you can fork it and
+actually make it happen.

lwan-serve-files.c

@@ -34,9 +34,11 @@
 static lwan_http_status_t
 _serve_file_stream(lwan_t* l, lwan_request_t *request, void *data)
 {
+ char headers[512];
  lwan_http_status_t return_status;
  int file_fd;
  struct stat st;
+ size_t header_len;
 
  if (UNLIKELY((file_fd = open(data, O_RDONLY)) < 0)) {
  return_status = (errno == EACCES) ? HTTP_FORBIDDEN : HTTP_NOT_FOUND;
@@ -63,15 +65,27 @@ _serve_file_stream(lwan_t* l, lwan_request_t *request, void *data)
  return _serve_file_stream(l, request, index_file);
  }
 
- lwan_request_set_corked(request, true);
  request->response->content_length = st.st_size;
- if (UNLIKELY(!lwan_response_header(l, request, HTTP_OK))) {
+ header_len = lwan_prepare_response_header(l, request, HTTP_OK, headers);
+ if (!header_len) {
  return_status = HTTP_INTERNAL_ERROR;
- goto end_corked;
+ goto end;
  }
 
  if (request->method == HTTP_HEAD) {
- return_status = HTTP_OK;
+ if (UNLIKELY(write(request->fd, headers, header_len) < 0)) {
+ perror("write");
+ return_status = HTTP_INTERNAL_ERROR;
+ } else
+ return_status = HTTP_OK;
+ goto end;
+ }
+
+ lwan_request_set_corked(request, true);
+
+ if (UNLIKELY(write(request->fd, headers, header_len) < 0)) {
+ perror("write");
+ return_status = HTTP_INTERNAL_ERROR;
  goto end_corked;
  }
 

lwan.c

@@ -35,6 +35,7 @@
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/types.h>
+#include <sys/uio.h>
 #include <unistd.h>
 
 #include "lwan.h"
@@ -596,10 +597,11 @@ lwan_request_set_response(lwan_request_t *request, lwan_response_t *response)
 #define APPEND_CONSTANT(const_str_) \
  APPEND_STRING_LEN((const_str_), sizeof(const_str_) - 1)
 
-bool
-lwan_response_header(lwan_t *l __attribute__((unused)), lwan_request_t *request, lwan_http_status_t status)
+ALWAYS_INLINE size_t
+lwan_prepare_response_header(lwan_t *l __attribute__((unused)), lwan_request_t *request, lwan_http_status_t status,
+ char headers[])
 {
- char headers[512], *p_headers;
+ char *p_headers;
  char buffer[32];
  int32_t len;
 
@@ -620,12 +622,7 @@ lwan_response_header(lwan_t *l __attribute__((unused)), lwan_request_t *request,
  (request->flags.is_keep_alive ? sizeof("Keep-Alive") : sizeof("Close")) - 1);
  APPEND_CONSTANT("\r\n\r\n\0");
 
- if (UNLIKELY(write(request->fd, headers, strlen(headers)) < 0)) {
- perror("write header");
- return false;
- }
-
- return true;
+ return p_headers - headers - 1;
 }
 
 #undef APPEND_STRING_LEN
@@ -637,6 +634,8 @@ lwan_response_header(lwan_t *l __attribute__((unused)), lwan_request_t *request,
 bool
 lwan_response(lwan_t *l, lwan_request_t *request, lwan_http_status_t status)
 {
+ char headers[512];
+
  if (UNLIKELY(!request->response)) {
  lwan_default_response(l, request, status);
  return false;
@@ -654,16 +653,25 @@ lwan_response(lwan_t *l, lwan_request_t *request, lwan_http_status_t status)
  return false;
  }
 
- if (UNLIKELY(!lwan_response_header(l, request, status)))
- return false;
+ size_t header_len = lwan_prepare_response_header(l, request, status, headers);
+ if (!header_len)
+ return lwan_default_response(l, request, HTTP_INTERNAL_ERROR);
 
- if (request->method == HTTP_HEAD)
+ if (request->method == HTTP_HEAD) {
+ if (write(request->fd, headers, header_len) < 0) {
+ perror("write");
+ return false;
+ }
  return true;
+ }
+
+ struct iovec response_vec[] = {
+ { .iov_base = headers, .iov_len = header_len },
+ { .iov_base = request->response->content, .iov_len = request->response->content_length }
+ };
 
- if (UNLIKELY(write(request->fd,
- request->response->content,
- request->response->content_length) < 0)) {
- perror("write response");
+ if (UNLIKELY(writev(request->fd, response_vec, N_ELEMENTS(response_vec)) < 0)) {
+ perror("writev");
  return false;
  }
 

lwan.h

@@ -172,7 +172,7 @@ void lwan_set_url_map(lwan_t *l, lwan_url_map_t *url_map);
 void lwan_main_loop(lwan_t *l);
 void lwan_request_set_response(lwan_request_t *request, lwan_response_t *response);
 bool lwan_response(lwan_t *l, lwan_request_t *request, lwan_http_status_t status);
-bool lwan_response_header(lwan_t *l, lwan_request_t *request, lwan_http_status_t status);
+size_t lwan_prepare_response_header(lwan_t *l, lwan_request_t *request, lwan_http_status_t status, char header_buffer[]);
 bool lwan_default_response(lwan_t *l, lwan_request_t *request, lwan_http_status_t status);
 const char *lwan_http_status_as_string(lwan_http_status_t status) __attribute__((pure));
 const char *lwan_determine_mime_type_for_file_name(char *file_name) __attribute__((pure));