Confused about my performance test.

[FH0]

I guess tokio is 5% slower than native epoll, but actually 30%. I don't know the reason, can you help me?

tokio version: v0.3.3
platform: Debian-10 x86_64

native epoll

#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>

void epoll_add(int epoll_fd, int fd, uint32_t event) {
  struct epoll_event ev;
  ev.events = event;
  ev.data.fd = fd;
  epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ev);
}

void epoll_del(int epoll_fd, int fd) {
  epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, NULL);
}

void set_non_blocking(int fd) {
  int previous = fcntl(fd, F_GETFL, 0);
  assert(previous != -1);
  assert(fcntl(fd, F_SETFL, previous | O_NONBLOCK) != -1);
}

int main(int argc, char const *argv[]) {
  int listen_socket = socket(AF_INET, SOCK_STREAM, 0);
  int enable = 1;
  setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));

  struct sockaddr_in listen_addr;
  listen_addr.sin_family = AF_INET;
  listen_addr.sin_port = htons(8888);
  memset(&listen_addr.sin_addr, 0, sizeof(listen_addr.sin_addr));
  bind(listen_socket, (const struct sockaddr *)&listen_addr, sizeof(listen_addr));

  set_non_blocking(listen_socket);

  listen(listen_socket, 512);

  int epoll_fd = epoll_create1(0);
  epoll_add(epoll_fd, listen_socket, EPOLLIN);

  for (;;) {
    struct epoll_event evs[512];
    int event_num = epoll_wait(epoll_fd, evs, 512, -1);
    for (int i = 0; i < event_num; ++i) {
      if (evs[i].data.fd == listen_socket) {
        struct sockaddr_in connect_addr;
        socklen_t addr_len = sizeof(connect_addr);
        int connect_socket = accept(listen_socket, (struct sockaddr *)&connect_addr, &addr_len);
        set_non_blocking(connect_socket);
        epoll_add(epoll_fd, connect_socket, EPOLLIN);
      } else if (evs[i].events & EPOLLIN) {
        for (;;) {
          char buf[512];
          ssize_t nread = read(evs[i].data.fd, buf, sizeof(buf));

          if (nread == 0) {
            close(evs[i].data.fd);
            goto out;
          } else if (nread == -1) {
            if (errno == EAGAIN || errno == EWOULDBLOCK) {
              break;
            } else {
              printf("unexpected\n");
              abort();
            }
          }
        }

        if (write(evs[i].data.fd, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 1\r\n\r\na", 65) <= 0) {
          perror("write");
          abort();
        }
        close(evs[i].data.fd);
        epoll_del(epoll_fd, evs[i].data.fd);

      out:;
      } else {
        printf("unexpected\n");
        abort();
      }
    }
  }

  return 0;
}

tokio

use tokio::net::TcpListener;
use tokio::prelude::*;

#[tokio::main]
async fn main() {
    let listener = TcpListener::bind("0.0.0.0:8887").await.unwrap();

    loop {
        let (mut socket, _) = listener.accept().await.unwrap();

        tokio::spawn(async move {
            let mut buf = [0; 1024];

            loop {
                match socket.read(&mut buf).await {
                    Ok(n) => {
                        if n == 0 {
                            return
                        }
                    },
                    Err(e) => {
                        eprintln!("failed to read from socket; err = {:?}", e);
                        return;
                    }
                };

                if let Err(e) = socket.write_all("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 1\r\n\r\na".as_bytes()).await {
                    eprintln!("failed to write to socket; err = {:?}", e);
                }
                return;
            }
        });
    }
}

Build native epoll bygcc benchmark.c -O3 -Wall
Buils tokio bycargo build --release
Test native epoll by ab -c 50 -n 10000 http://127.0.0.1:8888/
Test tokio by ab -c 50 -n 10000 http://127.0.0.1:8887/

Native epoll result

This is ApacheBench, Version 2.3 <$Revision: 1843412 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 127.0.0.1 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests


Server Software:
Server Hostname:        127.0.0.1
Server Port:            8888

Document Path:          /
Document Length:        1 bytes

Concurrency Level:      50
Time taken for tests:   0.676 seconds
Complete requests:      10000
Failed requests:        0
Total transferred:      650000 bytes
HTML transferred:       10000 bytes
Requests per second:    14792.97 [#/sec] (mean)
Time per request:       3.380 [ms] (mean)
Time per request:       0.068 [ms] (mean, across all concurrent requests)
Transfer rate:          939.01 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    2   0.2      2       3
Processing:     1    2   0.2      2       3
Waiting:        0    1   0.4      1       3
Total:          2    3   0.2      3       5

Percentage of the requests served within a certain time (ms)
  50%      3
  66%      3
  75%      3
  80%      3
  90%      4
  95%      4
  98%      4
  99%      4
 100%      5 (longest request)

Tokio result

This is ApacheBench, Version 2.3 <$Revision: 1843412 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 127.0.0.1 (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests


Server Software:
Server Hostname:        127.0.0.1
Server Port:            8887

Document Path:          /
Document Length:        1 bytes

Concurrency Level:      50
Time taken for tests:   0.888 seconds
Complete requests:      10000
Failed requests:        0
Total transferred:      650000 bytes
HTML transferred:       10000 bytes
Requests per second:    11256.95 [#/sec] (mean)
Time per request:       4.442 [ms] (mean)
Time per request:       0.089 [ms] (mean, across all concurrent requests)
Transfer rate:          714.55 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    2   0.4      2       4
Processing:     1    2   0.4      2       4
Waiting:        0    1   0.5      1       4
Total:          2    4   0.3      4       6

Percentage of the requests served within a certain time (ms)
  50%      4
  66%      4
  75%      5
  80%      5
  90%      5
  95%      5
  98%      5
  99%      5
 100%      6 (longest request)

I came to the conclusion through rps.

[Darksonn]

What happens if you change the Tokio code to this?

use tokio::net::TcpListener;
use tokio::prelude::*;

#[tokio::main]
async fn main() {
    tokio::spawn(real_main()).await.unwrap();
}

async fn real_main() {
    let listener = TcpListener::bind("0.0.0.0:8887").await.unwrap();

    loop {
        let (mut socket, _) = listener.accept().await.unwrap();

        tokio::spawn(async move {
            let mut buf = [0; 1024];

            loop {
                match socket.read(&mut buf).await {
                    Ok(n) => {
                        if n == 0 {
                            return
                        }
                    },
                    Err(e) => {
                        eprintln!("failed to read from socket; err = {:?}", e);
                        return;
                    }
                };

                if let Err(e) = socket.write_all("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 1\r\n\r\na".as_bytes()).await {
                    eprintln!("failed to write to socket; err = {:?}", e);
                }
                return;
            }
        });
    }
}

[FH0]

What happens if you change the Tokio code to this?

use tokio::net::TcpListener;
use tokio::prelude::*;

#[tokio::main]
async fn main() {
    tokio::spawn(real_main()).await.unwrap();
}

async fn real_main() {
    let listener = TcpListener::bind("0.0.0.0:8887").await.unwrap();

    loop {
        let (mut socket, _) = listener.accept().await.unwrap();

        tokio::spawn(async move {
            let mut buf = [0; 1024];

            loop {
                match socket.read(&mut buf).await {
                    Ok(n) => {
                        if n == 0 {
                            return
                        }
                    },
                    Err(e) => {
                        eprintln!("failed to read from socket; err = {:?}", e);
                        return;
                    }
                };

                if let Err(e) = socket.write_all("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 1\r\n\r\na".as_bytes()).await {
                    eprintln!("failed to write to socket; err = {:?}", e);
                }
                return;
            }
        });
    }
}

see my the other discussion. #3155