From: Alexey K. <ale...@or...> - 2013-10-28 14:18:46
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- runtest/tcp_cmds | 1 + testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/README | 16 + testcases/network/tcp_fastopen/tcp_fastopen.c | 823 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 190 +++++ 6 files changed, 1055 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/README create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/runtest/tcp_cmds b/runtest/tcp_cmds index 4722ca8..68d1ede 100644 --- a/runtest/tcp_cmds +++ b/runtest/tcp_cmds @@ -20,3 +20,4 @@ tcpdump tcpdump01 telnet telnet01 iptables iptables_tests.sh dhcpd dhcpd_tests.sh +tcp_fastopen tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..bff25e5 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/README b/testcases/network/tcp_fastopen/README new file mode 100644 index 0000000..4e19add --- /dev/null +++ b/testcases/network/tcp_fastopen/README @@ -0,0 +1,16 @@ +TCP Fast Open (TFO) is an extension to speed up the opening of TCP connections +between two endpoints. It reduces the number of round time trips (RTT) +required in TCP conversations. + +The test must be run on Linux versions higher then 3.7. (client side +implemented in Linux 3.6, server side in Linux 3.7). + +Developers reported (in "TCP Fast Open", CoNEXT 2011), TFO could result in +speed improvements of between 4% and 41% in the page load times on popular web +sites. + +With the default test scenario, a client is constantly sending requests to +a server and waiting for replies. The server is closing a connection after 3 +requests from the client. This scenario was chosen to simulate an average +conversation between a web-browser and an application server. So the test +results with TFO support enabled must be at least 3 percent faster. diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..c698628 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,823 @@ +/* + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/sysinfo.h> +#include <sys/poll.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; + +enum { + TCP_NEW_API = 0, + TCP_OLD_API +}; +static int tcp_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; + +static int sfd; + +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static int client_byte = 1; +static int server_byte = 2; +static const int start_byte = 0x24; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +static char *rpath = "./tfo_result"; + +static int main_pid; + +socklen_t sock_len, csock_len; + +static int force_run; + +/* test options */ +static char *narg, *Narg, *qarg, *barg, *Barg, + *rarg, *Rarg, *aarg, *Targ; +static int nflag, Nflag, qflag, bflag, Bflag, dflag, + rflag, Rflag, aflag, Hflag, Tflag, gflag; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* common params */ +static int skip_cleanup; +static int verbose; + +static const option_t options[] = { + /* server params */ + {"R:", &Rflag, &Rarg}, + {"q:", &qflag, &qarg}, + + /* client params */ + {"H:", &Hflag, &server_addr}, + {"a:", &aflag, &aarg}, + {"n:", &nflag, &narg}, + {"b:", &bflag, &barg}, + {"N:", &Nflag, &Narg}, + {"B:", &Bflag, &Barg}, + {"T:", &Tflag, &Targ}, + {"r:", &rflag, &rarg}, + {"d:", &dflag, &rpath}, + + /* common */ + {"g:", &gflag, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &tcp_api, NULL}, + {"O", &tfo_support, NULL}, + {"s", &skip_cleanup, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -s Skip cleanup\n"); + printf(" -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -b x x is a byte of client message\n"); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -B x x is a byte of server message\n"); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 + +/* current thread size */ +static pthread_attr_t attr; +static pthread_t *thread_ids; +static int threads_num; +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void cleanup(void) +{ + static int first = 1; + if (!first) + return; + first = 0; + + tst_resm(TINFO, "cleanup"); + + free(client_msg); + free(server_msg); + + if (skip_cleanup) + return; + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} + +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = *fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", + __FILE__, __LINE__); + break; + } + if (ret == 0) { + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(*fd, buf + *offset, + *buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && (errno == EINTR || + errno == EWOULDBLOCK || errno == EAGAIN)) + continue; + else + break; + } + + if (len == 0) + tst_resm(TINFO, "sock was closed '%d'", *fd); + + return len; +} + +void client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + + len = sock_recv_poll(fd, buf, &server_msg_size, &offset); + + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_byte + 1)) { + tst_resm(TFAIL | TERRNO, + "recv failed, sock '%d'", *fd); + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + break; + } + + offset += len; + + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + if (buf[0] == start_byte + 1) { + /* recv last msg, close socket */ + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + } + break; + } +} + +int client_connect_send(int *cfd) +{ + *cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(*cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (*cfd == -1) { + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", + __FILE__, __LINE__); + return -1; + } + + if (tcp_api == TCP_NEW_API) { + /* Replaces connect() + send()/write() */ + if (sendto(*cfd, client_msg, client_msg_size, + MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != client_msg_size) { + tst_resm(TFAIL | TERRNO, "sendto failed"); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(*cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + tst_resm(TFAIL | TERRNO, "connect failed"); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + + if (send(*cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, + "send failed on sock '%d'", *cfd); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + } + + return 0; +} + +void *client_fn(void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + + /* connect & send requests */ + if (client_connect_send(&cfd)) + return NULL; + client_recv(&cfd, buf); + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + + if (client_connect_send(&cfd)) + return NULL; + client_recv(&cfd, buf); + + continue; + + } else if (ret > 0) { + tst_resm_hexd(TFAIL, buf, 1, + "received after recv, sock '%d':", cfd); + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, "send failed"); + break; + } + client_recv(&cfd, buf); + } + + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + return NULL; +} + +union un_uint16 { + char b[2]; + uint16_t val; +}; + +static struct timeval tv_client_start; +static struct timeval tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + client_msg[0] = start_byte; + client_msg[1] = server_byte; + + union un_uint16 un; + un.val = htons(server_msg_size); + memcpy(client_msg + 2, un.b, 2); + client_msg[client_msg_size - 1] = end_byte; + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + gettimeofday(&tv_client_start, NULL); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + ++threads_num; + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + + int i; + for (i = 0; i < clients_num; ++i) + pthread_join(thread_ids[i], &res); + + threads_num = 0; + + gettimeofday(&tv_client_end, NULL); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + + tv_client_end.tv_usec - tv_client_start.tv_usec; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); + + /* ask server to terminate */ + int cfd; + client_msg[0] = start_byte + 1; + if (!client_connect_send(&cfd)) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); +} + +static void client_cleanup(void) +{ + void *res = NULL; + int i; + for (i = 0; i < threads_num; ++i) + pthread_join(thread_ids[i], &res); + + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + +void make_server_reply(const char *buf, char **msg, uint16_t *size) +{ + union un_uint16 un; + memcpy(un.b, buf + 2, 2); + *size = ntohs(un.val); + if (*size < 2 || *size > max_msg_len) + *size = 2; + + if (msg == NULL) { + tst_resm(TWARN, "failed to make a reply"); + return; + } + + *msg = SAFE_MALLOC(NULL, *size); + memset(*msg, buf[1], *size - 1); + (*msg)[*size - 1] = end_byte; +} + +void *server_fn(void *cfd) +{ + int *client_fd = cfd; + int num_requests = 0, offset = 0; + uint16_t reply_size = 0; + char *msg = NULL, buf[max_msg_len]; + setsockopt(*client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t len; + + while (1) { + errno = 0; + len = sock_recv_poll(client_fd, buf, &max_msg_len, &offset); + + if (len == 0) { + break; + } else if (len < 0 || (offset + len) > max_msg_len || + (buf[0] != start_byte && buf[0] != start_byte + 1)) { + tst_resm(TFAIL, "recv failed, sock '%d'", *client_fd); + break; + } + + offset += len; + + if (buf[offset - 1] != end_byte) { + tst_resm(TINFO, "msg is not complete, continue recv"); + continue; + } + + if (buf[0] == start_byte + 1) + tst_brkm(TBROK, cleanup, "client asks to terminate..."); + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *client_fd); + } + + if (msg == NULL) + make_server_reply(buf, &msg, &reply_size); + + ++num_requests; + + /* if last message */ + if (num_requests >= server_max_requests) + msg[0] = start_byte + 1; + else + msg[0] = start_byte; + + if (send(*client_fd, msg, reply_size, MSG_NOSIGNAL) == -1) + tst_resm(TWARN | TERRNO, "Error while sending msg"); + else { + if (verbose) { + tst_resm_hexd(TINFO, msg, reply_size, + "msg sent:"); + } + } + + offset = 0; + + if (num_requests >= server_max_requests) { + if (verbose) + tst_resm(TINFO, "Max reqs, close socket"); + shutdown(*client_fd, SHUT_WR); + break; + } + } + + free(msg); + SAFE_CLOSE(NULL, *client_fd); + free(client_fd); + + return NULL; +} + +static void server_thread_add(int *client_fd) +{ + int *cfd = SAFE_MALLOC(NULL, sizeof(int)); + *cfd = *client_fd; + pthread_t id; + if (pthread_create(&id, &attr, server_fn, cfd) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + sleep(1); + } + tst_resm(TINFO, "the name assigned"); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (tcp_api == TCP_NEW_API) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); + if (local_addrinfo) + freeaddrinfo(local_addrinfo); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * ones a thread finishs its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetached failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) { + tst_brkm(TBROK, cleanup, "Can't create client socket"); + continue; + } + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(&client_fd); + } +} + +static void check_opt(const char *name, int *flag, + char *arg, int *val, int lim) +{ + if (*flag) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, int *flag, + char *arg, long *val, long lim) +{ + if (*flag) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +/* cleanup flags */ +void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client num is not set, use num of processors */ + struct sysinfo si; + if (sysinfo(&si) == 0) + clients_num = si.procs; + + check_opt("a", &aflag, aarg, &clients_num, 1); + check_opt("r", &rflag, rarg, &client_max_requests, 1); + check_opt("R", &Rflag, Rarg, &server_max_requests, 1); + check_opt("n", &nflag, narg, &client_msg_size, 1); + + check_opt("b", &bflag, barg, &client_byte, 0); + if (client_byte == start_byte || client_byte == end_byte) + tst_brkm(TBROK, NULL, "-b option, wrong arg"); + + check_opt("N", &Nflag, Narg, &server_msg_size, 1); + + check_opt("B", &Bflag, Barg, &server_byte, 0); + if (server_byte == start_byte || server_byte == end_byte) + tst_brkm(TBROK, NULL, "-B option, wrong arg"); + + check_opt("q", &qflag, qarg, &tfo_queue_size, 1); + check_opt_l("T", &Tflag, Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(cleanup, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + main_pid = getpid(); + tst_resm(TINFO, "pid '%d'", main_pid); + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (tcp_api == TCP_NEW_API) ? "new" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + tst_resm(TINFO, "client msg byte: %02x", client_byte); + tst_resm(TINFO, "server msg byte: %02x", server_byte); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +static void test_run() +{ + tcp.run(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + test_run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..d56c510 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +client_requests=2000000 +server_port=$[ $RANDOM % 28232 + 32768 ] + +while getopts :hu:H:sr:p: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "r x the number of requests" + echo "p x server port" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + r) client_requests=$OPTARG ;; + p) server_port=$OPTARG ;; + *) + tst_brkm TBROK NULL "unknown option: $opt" + exit 2 + ;; + esac +done + +run_remote_cmd() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + + if [ "$use_ssh" = 1 ]; then + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" + else + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" + fi +} + +cleanup() +{ + rm -f $tfo_result + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" + # remove test files on remote host + sleep 2 + run_remote_cmd "[ -e ${tdir}tcp_fastopen ] && rm -f ${tdir}tcp_fastopen" + sleep 2 +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK NULL "Failed to read result file" + exit 2 + fi + else + tst_brkm TBROK NULL "Failed to find result file" + exit 2 + fi +} + +check_exit_status() +{ + if [ "$1" -ne "0" ]; then + tst_brkm TBROK NULL "Last test has failed" + exit $1; + fi +} + +export RC=0 +export TST_TOTAL=1 +export TCID="tcp_fastopen" +export TST_COUNT=0 + +tfo_result_ms=0 +bind_timeout=10 +clients_num=2 +max_requests=3 +msg_timeout=10000000 + +# Setup +type tst_resm > /dev/null 2>&1 +if [ $? -eq 1 ]; then + echo "$TCID 0 TCONF : failed to find LTP tst_* utilities" + exit 2 +fi + +tst_kvercmp 3 7 0 +if [ $? -eq 0 ]; then + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" + exit 0 +fi + +if [ -z $remote_addr ]; then + tst_brkm TBROK NULL "you must specify server address" + exit 2 +fi + + +if [ -z $LTPROOT ]; then + tdir="./" +else + tdir=${LTPROOT}/testcases/bin/ +fi + +if [ -z "$TMPDIR" ]; then + tfo_result="${tdir}tfo_result" +else + tfo_result="${TMPDIR}/tfo_result" +fi + +trap "cleanup" EXIT + +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" + +if [ "$use_ssh" = 1 ]; then + scp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null +else + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null +fi + + +# Run +tcp_api_opt=("-o -O" "") +vtime=(0 0) + +for (( i = 0; i < 2; ++i )); do + # kill tcp server on remote machine + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" + + sleep 2 + + # run tcp server on remote machine + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests -q 100 \ +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" + + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-n 32 -b 23 -N 128 -B 8 -T $msg_timeout -H $remote_addr \ +${tcp_api_opt[$i]} -g $server_port -d $tfo_result + + check_exit_status $? + + vtime[$i]=`read_result_file` + + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" + exit 2 + fi + server_port=$[ $server_port + 1 ] +done + +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] + +if (( $tfo_cmp < 3 )); then + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" + exit 1 +fi + +tst_resm TPASS "TFO performance result is $tfo_cmp percent" +exit 0 -- 1.7.1 |
From: Hangbin L. <liu...@gm...> - 2013-10-29 03:17:08
|
On Mon, Oct 28, 2013 at 06:18:14PM +0400, Alexey Kodanev wrote: > This is a perfomance test for TCP Fast Open (TFO) which is an extension to > speed up the opening of TCP connections between two endpoints. It reduces > the number of round time trips (RTT) required in TCP conversations. TFO could > result in speed improvements of between 4% and 41% in the page load times on > popular web sites. > > The default test scenario simulates an average conversation between a > web-browser and an application server, so the test results with TFO enabled > must be at least 3 percent faster. > > The test must be run on Linux versions higher then 3.7. (TFO client side > implemented in Linux 3.6, server side in Linux 3.7). > > Signed-off-by: Alexey Kodanev <ale...@or...> > --- > runtest/tcp_cmds | 1 + > testcases/network/tcp_fastopen/.gitignore | 1 + > testcases/network/tcp_fastopen/Makefile | 24 + > testcases/network/tcp_fastopen/README | 16 + > testcases/network/tcp_fastopen/tcp_fastopen.c | 823 ++++++++++++++++++++ > testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 190 +++++ > 6 files changed, 1055 insertions(+), 0 deletions(-) > create mode 100644 testcases/network/tcp_fastopen/.gitignore > create mode 100644 testcases/network/tcp_fastopen/Makefile > create mode 100644 testcases/network/tcp_fastopen/README > create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c > create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh > > diff --git a/runtest/tcp_cmds b/runtest/tcp_cmds > index 4722ca8..68d1ede 100644 > --- a/runtest/tcp_cmds > +++ b/runtest/tcp_cmds > @@ -20,3 +20,4 @@ tcpdump tcpdump01 > telnet telnet01 > iptables iptables_tests.sh > dhcpd dhcpd_tests.sh > +tcp_fastopen tcp_fastopen_run.sh > diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore > new file mode 100644 > index 0000000..f321cf0 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/.gitignore > @@ -0,0 +1 @@ > +/tcp_fastopen > diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile > new file mode 100644 > index 0000000..bff25e5 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/Makefile > @@ -0,0 +1,24 @@ > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + > +top_srcdir ?= ../../.. > + > +include $(top_srcdir)/include/mk/testcases.mk > + > +INSTALL_TARGETS := tcp_fastopen_run.sh > +LDLIBS += -lpthread > + > +include $(top_srcdir)/include/mk/generic_leaf_target.mk > diff --git a/testcases/network/tcp_fastopen/README b/testcases/network/tcp_fastopen/README > new file mode 100644 > index 0000000..4e19add > --- /dev/null > +++ b/testcases/network/tcp_fastopen/README > @@ -0,0 +1,16 @@ > +TCP Fast Open (TFO) is an extension to speed up the opening of TCP connections > +between two endpoints. It reduces the number of round time trips (RTT) > +required in TCP conversations. > + > +The test must be run on Linux versions higher then 3.7. (client side > +implemented in Linux 3.6, server side in Linux 3.7). > + > +Developers reported (in "TCP Fast Open", CoNEXT 2011), TFO could result in > +speed improvements of between 4% and 41% in the page load times on popular web > +sites. > + > +With the default test scenario, a client is constantly sending requests to > +a server and waiting for replies. The server is closing a connection after 3 > +requests from the client. This scenario was chosen to simulate an average > +conversation between a web-browser and an application server. So the test > +results with TFO support enabled must be at least 3 percent faster. > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c > new file mode 100644 > index 0000000..c698628 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c > @@ -0,0 +1,823 @@ > +/* > + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > + * > + * This program is free software; you can redistribute it and/or > + * modify it under the terms of the GNU General Public License as > + * published by the Free Software Foundation; either version 2 of > + * the License, or (at your option) any later version. > + * > + * This program is distributed in the hope that it would be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, write the Free Software Foundation, > + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + * > + * Author: Alexey Kodanev <ale...@or...> > + * > + */ > + > +#include <sys/stat.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <sys/time.h> > +#include <sys/resource.h> > +#include <sys/sysinfo.h> > +#include <sys/poll.h> > +#include <netinet/in.h> > +#include <arpa/inet.h> > +#include <netdb.h> > +#include <signal.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <pthread.h> > +#include <unistd.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <unistd.h> > +#include <string.h> > +#include <errno.h> > + > +#include "test.h" > +#include "usctest.h" > +#include "safe_macros.h" > + > +char *TCID = "tcp_fastopen"; > + > +static const int max_msg_len = 1500; > + > +/* TCP server requiers */ > +#ifndef TCP_FASTOPEN > +#define TCP_FASTOPEN 23 > +#endif > + > +/* TCP client requiers */ > +#ifndef MSG_FASTOPEN > +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ > +#endif > + > +enum { > + TCP_SERVER = 0, > + TCP_CLIENT, > +}; > +static int tcp_mode; > + > +enum { > + TFO_ENABLED = 0, > + TFO_DISABLED, > +}; > +static int tfo_support; > + > +enum { > + TCP_NEW_API = 0, > + TCP_OLD_API > +}; > +static int tcp_api; > + > +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; > +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; > +static int tw_reuse_changed; > +static int tfo_cfg_value; > +static int tfo_bit_num; > +static int tfo_cfg_changed; > + > +static int sfd; > + > +static int tfo_queue_size = 100; > +static int max_queue_len = 100; > +static int client_byte = 1; > +static int server_byte = 2; > +static const int start_byte = 0x24; > +static const int end_byte = 0x0a; > +static int client_msg_size = 32; > +static int server_msg_size = 128; > +static char *client_msg; > +static char *server_msg; > + > +/* > + * The number of requests from client after > + * which server has to close the connection. > + */ > +static int server_max_requests = 3; > +static int client_max_requests = 10; > +static int clients_num = 2; > +static char *tcp_port = "61000"; > +static char *server_addr = "localhost"; > +static char *rpath = "./tfo_result"; > + > +static int main_pid; > + > +socklen_t sock_len, csock_len; > + > +static int force_run; > + > +/* test options */ > +static char *narg, *Narg, *qarg, *barg, *Barg, > + *rarg, *Rarg, *aarg, *Targ; > +static int nflag, Nflag, qflag, bflag, Bflag, dflag, > + rflag, Rflag, aflag, Hflag, Tflag, gflag; > + > +/* how long a client must wait for the server's reply, microsec */ > +static long wait_timeout = 10000000; > + > +/* common params */ > +static int skip_cleanup; > +static int verbose; > + > +static const option_t options[] = { > + /* server params */ > + {"R:", &Rflag, &Rarg}, > + {"q:", &qflag, &qarg}, > + > + /* client params */ > + {"H:", &Hflag, &server_addr}, > + {"a:", &aflag, &aarg}, > + {"n:", &nflag, &narg}, > + {"b:", &bflag, &barg}, > + {"N:", &Nflag, &Narg}, > + {"B:", &Bflag, &Barg}, > + {"T:", &Tflag, &Targ}, > + {"r:", &rflag, &rarg}, > + {"d:", &dflag, &rpath}, > + > + /* common */ > + {"g:", &gflag, &tcp_port}, > + {"F", &force_run, NULL}, > + {"l", &tcp_mode, NULL}, > + {"o", &tcp_api, NULL}, > + {"O", &tfo_support, NULL}, > + {"s", &skip_cleanup, NULL}, > + {"v", &verbose, NULL}, > + {NULL, NULL, NULL} > +}; > + > +static void help(void) > +{ > + printf("\n -s Skip cleanup\n"); > + printf(" -F Force to run\n"); > + printf(" -v Verbose\n"); > + printf(" -o Use old TCP API, default is new TCP API\n"); > + printf(" -O TFO support is off, default is on\n"); > + printf(" -l Become TCP Client, default is TCP server\n"); > + printf(" -g x x - server port, default is %s\n", tcp_port); > + > + printf("\n Client:\n"); > + printf(" -H x x - server name or ip address, default is '%s'\n", > + server_addr); > + printf(" -a x x - num of clients running in parallel\n"); > + printf(" -r x x - num of client requests\n"); > + printf(" -n x Client message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -b x x is a byte of client message\n"); > + printf(" -N x Server message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -B x x is a byte of server message\n"); > + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", > + wait_timeout); > + printf(" -d x x is a path to the file where results are saved\n"); > + > + printf("\n Server:\n"); > + printf(" -R x x - num of requests, after which conn. closed\n"); > + printf(" -q x x - server's limit on the queue of TFO requests\n"); > +} > + > +struct tcp_func { > + void (*init)(void); > + void (*run)(void); > + void (*cleanup)(void); > +}; > +static struct tcp_func tcp; > + > +#define MAX_THREADS 10000 > + > +/* current thread size */ > +static pthread_attr_t attr; > +static pthread_t *thread_ids; > +static int threads_num; > +static struct addrinfo *remote_addrinfo; > +static struct addrinfo *local_addrinfo; > +static const struct linger clo = { 1, 3 }; > + > +static void cleanup(void) > +{ > + static int first = 1; > + if (!first) > + return; > + first = 0; > + > + tst_resm(TINFO, "cleanup"); > + > + free(client_msg); > + free(server_msg); > + > + if (skip_cleanup) > + return; > + > + tcp.cleanup(); > + > + if (tfo_cfg_changed) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tfo_cfg_value &= ~tfo_bit_num; > + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); > + tst_resm(TINFO, "unset '%s' back to '%d'", > + tfo_cfg, tfo_cfg_value); > + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); > + } > + > + if (tw_reuse_changed) { > + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); > + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); > + } > + TEST_CLEANUP; > +} > + > +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) > +{ > + struct pollfd pfd; > + pfd.fd = *fd; > + pfd.events = POLLIN; > + int len = -1; > + while (1) { > + errno = 0; > + int ret = poll(&pfd, 1, wait_timeout / 1000); > + if (ret == -1) { > + if (errno == EINTR) > + continue; > + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", > + __FILE__, __LINE__); > + break; > + } > + if (ret == 0) { > + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); > + break; > + } > + > + if (ret != 1 || !(pfd.revents & POLLIN)) > + break; > + > + errno = 0; > + len = recv(*fd, buf + *offset, > + *buf_size - *offset, MSG_DONTWAIT); > + > + if (len == -1 && (errno == EINTR || > + errno == EWOULDBLOCK || errno == EAGAIN)) > + continue; > + else > + break; > + } > + > + if (len == 0) > + tst_resm(TINFO, "sock was closed '%d'", *fd); > + > + return len; > +} > + > +void client_recv(int *fd, char *buf) > +{ > + int len, offset = 0; > + > + while (1) { > + > + len = sock_recv_poll(fd, buf, &server_msg_size, &offset); > + > + /* socket closed or msg is not valid */ > + if (len < 1 || (offset + len) > server_msg_size || > + (buf[0] != start_byte && buf[0] != start_byte + 1)) { > + tst_resm(TFAIL | TERRNO, > + "recv failed, sock '%d'", *fd); > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) > + continue; > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *fd); > + } > + > + if (buf[0] == start_byte + 1) { > + /* recv last msg, close socket */ > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + } > + break; > + } > +} > + > +int client_connect_send(int *cfd) > +{ > + *cfd = socket(AF_INET, SOCK_STREAM, 0); > + const int flag = 1; > + setsockopt(*cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (*cfd == -1) { > + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", > + __FILE__, __LINE__); > + return -1; > + } > + > + if (tcp_api == TCP_NEW_API) { > + /* Replaces connect() + send()/write() */ > + if (sendto(*cfd, client_msg, client_msg_size, > + MSG_FASTOPEN | MSG_NOSIGNAL, > + remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "sendto failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } else { > + /* old TCP API */ > + if (connect(*cfd, remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen)) { > + tst_resm(TFAIL | TERRNO, "connect failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + > + if (send(*cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, > + "send failed on sock '%d'", *cfd); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } > + > + return 0; > +} > + > +void *client_fn(void *arg) > +{ > + char buf[server_msg_size]; > + int cfd, i; > + > + /* connect & send requests */ > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + for (i = 1; i < client_max_requests; ++i) { > + > + /* check connection, it can be closed */ > + int ret = 0; > + if (cfd != -1) > + ret = recv(cfd, buf, 1, MSG_DONTWAIT); > + > + if (ret == 0) { > + /* try to reconnect and send */ > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + continue; > + > + } else if (ret > 0) { > + tst_resm_hexd(TFAIL, buf, 1, > + "received after recv, sock '%d':", cfd); > + } > + > + if (verbose) { > + tst_resm_hexd(TINFO, client_msg, client_msg_size, > + "try to send msg[%d]", i); > + } > + > + if (send(cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "send failed"); > + break; > + } > + client_recv(&cfd, buf); > + } > + > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + return NULL; > +} > + > +union un_uint16 { > + char b[2]; > + uint16_t val; > +}; > + > +static struct timeval tv_client_start; > +static struct timeval tv_client_end; > + > +static void client_init(void) > +{ > + if (clients_num >= MAX_THREADS) { > + tst_brkm(TBROK, cleanup, > + "Unexpected num of clients '%d'", > + clients_num); > + } > + > + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); > + > + client_msg = SAFE_MALLOC(NULL, client_msg_size); > + memset(client_msg, client_byte, client_msg_size); > + client_msg[0] = start_byte; > + client_msg[1] = server_byte; > + > + union un_uint16 un; > + un.val = htons(server_msg_size); > + memcpy(client_msg + 2, un.b, 2); > + client_msg[client_msg_size - 1] = end_byte; > + > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + gettimeofday(&tv_client_start, NULL); > + int i; > + for (i = 0; i < clients_num; ++i) { > + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", > + __FILE__, __LINE__); > + } > + ++threads_num; > + } > +} > + > +static void client_run(void) > +{ > + void *res = NULL; > + long clnt_time = 0; > + > + int i; > + for (i = 0; i < clients_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + threads_num = 0; > + > + gettimeofday(&tv_client_end, NULL); > + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + > + tv_client_end.tv_usec - tv_client_start.tv_usec; > + > + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); > + > + /* ask server to terminate */ > + int cfd; > + client_msg[0] = start_byte + 1; > + if (!client_connect_send(&cfd)) { > + shutdown(cfd, SHUT_WR); > + SAFE_CLOSE(NULL, cfd); > + } > + > + /* the script tcp_fastopen_run.sh will remove it */ > + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); > +} > + > +static void client_cleanup(void) > +{ > + void *res = NULL; > + int i; > + for (i = 0; i < threads_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + free(thread_ids); > + > + if (remote_addrinfo) > + freeaddrinfo(remote_addrinfo); > +} > + > +void make_server_reply(const char *buf, char **msg, uint16_t *size) > +{ > + union un_uint16 un; > + memcpy(un.b, buf + 2, 2); > + *size = ntohs(un.val); > + if (*size < 2 || *size > max_msg_len) > + *size = 2; > + > + if (msg == NULL) { > + tst_resm(TWARN, "failed to make a reply"); > + return; > + } > + > + *msg = SAFE_MALLOC(NULL, *size); > + memset(*msg, buf[1], *size - 1); > + (*msg)[*size - 1] = end_byte; > +} > + > +void *server_fn(void *cfd) > +{ > + int *client_fd = cfd; > + int num_requests = 0, offset = 0; > + uint16_t reply_size = 0; > + char *msg = NULL, buf[max_msg_len]; > + setsockopt(*client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); > + ssize_t len; > + > + while (1) { > + errno = 0; > + len = sock_recv_poll(client_fd, buf, &max_msg_len, &offset); > + > + if (len == 0) { > + break; > + } else if (len < 0 || (offset + len) > max_msg_len || > + (buf[0] != start_byte && buf[0] != start_byte + 1)) { > + tst_resm(TFAIL, "recv failed, sock '%d'", *client_fd); > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) { > + tst_resm(TINFO, "msg is not complete, continue recv"); > + continue; > + } > + > + if (buf[0] == start_byte + 1) > + tst_brkm(TBROK, cleanup, "client asks to terminate..."); > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *client_fd); > + } > + > + if (msg == NULL) > + make_server_reply(buf, &msg, &reply_size); > + > + ++num_requests; > + > + /* if last message */ > + if (num_requests >= server_max_requests) > + msg[0] = start_byte + 1; > + else > + msg[0] = start_byte; > + > + if (send(*client_fd, msg, reply_size, MSG_NOSIGNAL) == -1) > + tst_resm(TWARN | TERRNO, "Error while sending msg"); > + else { > + if (verbose) { > + tst_resm_hexd(TINFO, msg, reply_size, > + "msg sent:"); > + } > + } > + > + offset = 0; > + > + if (num_requests >= server_max_requests) { > + if (verbose) > + tst_resm(TINFO, "Max reqs, close socket"); > + shutdown(*client_fd, SHUT_WR); > + break; > + } > + } > + > + free(msg); > + SAFE_CLOSE(NULL, *client_fd); > + free(client_fd); > + > + return NULL; > +} > + > +static void server_thread_add(int *client_fd) > +{ > + int *cfd = SAFE_MALLOC(NULL, sizeof(int)); > + *cfd = *client_fd; > + pthread_t id; > + if (pthread_create(&id, &attr, server_fn, cfd) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", __FILE__, __LINE__); > + } > +} > + > +static void server_init(void) > +{ > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + hints.ai_flags = AI_PASSIVE; > + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + sfd = socket(AF_INET, SOCK_STREAM, 0); > + if (sfd == -1) > + tst_brkm(TBROK, cleanup, "Failed to create a socket"); > + > + tst_resm(TINFO, "assigning a name to the server socket..."); > + if (!local_addrinfo) > + tst_brkm(TBROK, cleanup, "failed to get the address"); > + > + while (bind(sfd, local_addrinfo->ai_addr, > + local_addrinfo->ai_addrlen) == -1) { > + sleep(1); > + } > + tst_resm(TINFO, "the name assigned"); > + > + const int flag = 1; > + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (tcp_api == TCP_NEW_API) { > + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, > + sizeof(tfo_queue_size)) == -1) > + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); > + } > + > + listen(sfd, max_queue_len); > + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); > +} > + > +static void server_cleanup(void) > +{ > + SAFE_CLOSE(NULL, sfd); > + if (local_addrinfo) > + freeaddrinfo(local_addrinfo); > +} > + > +static void server_run(void) > +{ > + struct sockaddr_in client_addr; > + socklen_t addr_size = sizeof(client_addr); > + pthread_attr_init(&attr); > + > + /* > + * detaching threads allow to reclaim thread's resources > + * ones a thread finishs its work. > + */ > + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "setdetached failed"); > + > + while (1) { > + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, > + &addr_size); > + if (client_fd == -1) { > + tst_brkm(TBROK, cleanup, "Can't create client socket"); > + continue; > + } > + > + if (client_addr.sin_family == AF_INET) { > + if (verbose) { > + tst_resm(TINFO, "conn: port '%d', addr '%s'", > + client_addr.sin_port, > + inet_ntoa(client_addr.sin_addr)); > + } > + } > + server_thread_add(&client_fd); > + } > +} > + > +static void check_opt(const char *name, int *flag, > + char *arg, int *val, int lim) > +{ > + if (*flag) { > + if (sscanf(arg, "%i", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", > + name, lim); > + } > +} > + > +static void check_opt_l(const char *name, int *flag, > + char *arg, long *val, long lim) > +{ > + if (*flag) { > + if (sscanf(arg, "%ld", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", > + name, lim); > + } > +} > + > +/* cleanup flags */ > +void setup(int argc, char *argv[]) > +{ > + char *msg; > + msg = parse_opts(argc, argv, options, help); > + if (msg != NULL) > + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); > + > + /* if client num is not set, use num of processors */ > + struct sysinfo si; > + if (sysinfo(&si) == 0) > + clients_num = si.procs; > + > + check_opt("a", &aflag, aarg, &clients_num, 1); > + check_opt("r", &rflag, rarg, &client_max_requests, 1); > + check_opt("R", &Rflag, Rarg, &server_max_requests, 1); > + check_opt("n", &nflag, narg, &client_msg_size, 1); > + > + check_opt("b", &bflag, barg, &client_byte, 0); > + if (client_byte == start_byte || client_byte == end_byte) > + tst_brkm(TBROK, NULL, "-b option, wrong arg"); > + > + check_opt("N", &Nflag, Narg, &server_msg_size, 1); > + > + check_opt("B", &Bflag, Barg, &server_byte, 0); > + if (server_byte == start_byte || server_byte == end_byte) > + tst_brkm(TBROK, NULL, "-B option, wrong arg"); > + > + check_opt("q", &qflag, qarg, &tfo_queue_size, 1); > + check_opt_l("T", &Tflag, Targ, &wait_timeout, 0L); > + > + if (!force_run) > + tst_require_root(NULL); > + > + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { > + tst_brkm(TCONF, NULL, > + "Test must be run with kernel 3.7 or newer"); > + } > + > + /* check tcp fast open knob */ > + if (!force_run && access(tfo_cfg, F_OK) == -1) > + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); > + > + if (!force_run) { > + SAFE_FILE_SCANF(cleanup, tfo_cfg, "%d", &tfo_cfg_value); > + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); > + } > + > + tst_sig(FORK, DEF_HANDLER, cleanup); > + > + main_pid = getpid(); > + tst_resm(TINFO, "pid '%d'", main_pid); > + tst_resm(TINFO, "TCP %s is using %s TCP API.", > + (tcp_mode == TCP_SERVER) ? "server" : "client", > + (tcp_api == TCP_NEW_API) ? "new" : "old"); > + > + switch (tcp_mode) { > + case TCP_SERVER: > + tst_resm(TINFO, "max requests '%d'", > + server_max_requests); > + tcp.init = server_init; > + tcp.run = server_run; > + tcp.cleanup = server_cleanup; > + tfo_bit_num = 2; > + break; > + case TCP_CLIENT: > + tst_resm(TINFO, "connection: %s:%s", > + server_addr, tcp_port); > + tst_resm(TINFO, "client max req: %d", client_max_requests); > + tst_resm(TINFO, "clients num: %d", clients_num); > + tst_resm(TINFO, "client msg size: %d", client_msg_size); > + tst_resm(TINFO, "server msg size: %d", server_msg_size); > + tst_resm(TINFO, "client msg byte: %02x", client_byte); > + tst_resm(TINFO, "server msg byte: %02x", server_byte); > + > + tcp.init = client_init; > + tcp.run = client_run; > + tcp.cleanup = client_cleanup; > + tfo_bit_num = 1; > + break; > + } > + > + tfo_support = TFO_ENABLED == tfo_support; > + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { > + int value = (tfo_cfg_value & ~tfo_bit_num) > + | (tfo_support << (tfo_bit_num - 1)); > + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); > + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); > + tfo_cfg_changed = 1; > + } > + > + int reuse_value = 0; > + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); > + if (!reuse_value) { > + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); > + tw_reuse_changed = 1; > + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); > + } > + > + tst_resm(TINFO, "TFO support %s", > + (tfo_support) ? "enabled" : "disabled"); > + > + tcp.init(); > +} > + > +static void test_run() > +{ > + tcp.run(); > +} > + > +int main(int argc, char *argv[]) > +{ > + setup(argc, argv); > + > + test_run(); > + > + cleanup(); > + > + tst_exit(); > +} > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > new file mode 100755 > index 0000000..d56c510 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > @@ -0,0 +1,190 @@ > +#!/bin/bash > + > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > +# > +# Author: Alexey Kodanev <ale...@or...> > +# > + > +# default command-line options > +user_name="root" > +remote_addr=$RHOST > +use_ssh=0 > +client_requests=2000000 > +server_port=$[ $RANDOM % 28232 + 32768 ] > + > +while getopts :hu:H:sr:p: opt; do > + case "$opt" in > + h) > + echo "Usage:" > + echo "h help" > + echo "u x server user name" > + echo "H x server hostname or IP address" > + echo "s use ssh to run remote cmds" > + echo "r x the number of requests" > + echo "p x server port" > + exit 0 > + ;; > + u) user_name=$OPTARG ;; > + H) remote_addr=$OPTARG ;; > + s) use_ssh=1 ;; > + r) client_requests=$OPTARG ;; > + p) server_port=$OPTARG ;; > + *) > + tst_brkm TBROK NULL "unknown option: $opt" > + exit 2 > + ;; > + esac > +done > + > +run_remote_cmd() > +{ > + tst_resm TINFO "run cmd on $remote_addr: $1" > + > + if [ "$use_ssh" = 1 ]; then > + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" > + else > + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" > + fi > +} > + > +cleanup() > +{ > + rm -f $tfo_result > + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" > + # remove test files on remote host > + sleep 2 > + run_remote_cmd "[ -e ${tdir}tcp_fastopen ] && rm -f ${tdir}tcp_fastopen" > + sleep 2 > +} > + > +read_result_file() > +{ > + if [ -f $tfo_result ]; then > + if [ -r $tfo_result ]; then > + cat $tfo_result > + else > + tst_brkm TBROK NULL "Failed to read result file" > + exit 2 > + fi > + else > + tst_brkm TBROK NULL "Failed to find result file" > + exit 2 > + fi > +} > + > +check_exit_status() > +{ > + if [ "$1" -ne "0" ]; then > + tst_brkm TBROK NULL "Last test has failed" > + exit $1; > + fi > +} > + > +export RC=0 > +export TST_TOTAL=1 > +export TCID="tcp_fastopen" > +export TST_COUNT=0 > + > +tfo_result_ms=0 > +bind_timeout=10 > +clients_num=2 > +max_requests=3 > +msg_timeout=10000000 > + > +# Setup > +type tst_resm > /dev/null 2>&1 > +if [ $? -eq 1 ]; then > + echo "$TCID 0 TCONF : failed to find LTP tst_* utilities" > + exit 2 > +fi > + > +tst_kvercmp 3 7 0 > +if [ $? -eq 0 ]; then > + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" > + exit 0 > +fi > + > +if [ -z $remote_addr ]; then > + tst_brkm TBROK NULL "you must specify server address" > + exit 2 > +fi > + > + > +if [ -z $LTPROOT ]; then > + tdir="./" > +else > + tdir=${LTPROOT}/testcases/bin/ > +fi > + > +if [ -z "$TMPDIR" ]; then > + tfo_result="${tdir}tfo_result" > +else > + tfo_result="${TMPDIR}/tfo_result" > +fi > + > +trap "cleanup" EXIT > + > +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" > + > +if [ "$use_ssh" = 1 ]; then > + scp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null > +else > + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null > +fi > + > + > +# Run > +tcp_api_opt=("-o -O" "") > +vtime=(0 0) > + > +for (( i = 0; i < 2; ++i )); do > + # kill tcp server on remote machine > + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" > + > + sleep 2 > + > + # run tcp server on remote machine > + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests -q 100 \ > +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" > + > + sleep $bind_timeout > + > + # run local tcp client > + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ > +-n 32 -b 23 -N 128 -B 8 -T $msg_timeout -H $remote_addr \ > +${tcp_api_opt[$i]} -g $server_port -d $tfo_result > + > + check_exit_status $? > + > + vtime[$i]=`read_result_file` > + > + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then > + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" > + exit 2 > + fi > + server_port=$[ $server_port + 1 ] > +done > + > +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] > + > +if (( $tfo_cmp < 3 )); then > + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" > + exit 1 > +fi > + > +tst_resm TPASS "TFO performance result is $tfo_cmp percent" > +exit 0 > -- > 1.7.1 Tested on F18 and looks good. Acked-by Hangbin Liu <liu...@gm...> > > > ------------------------------------------------------------------------------ > October Webinars: Code for Performance > Free Intel webinars can help you accelerate application performance. > Explore tips for MPI, OpenMP, advanced profiling, and more. Get the most from > the latest Intel processors and coprocessors. See abstracts and register > > http://pubads.g.doubleclick.net/gampad/clk?id=60135991&iu=/4140/ostg.clktrk > _______________________________________________ > Ltp-list mailing list > Ltp...@li... > https://lists.sourceforge.net/lists/listinfo/ltp-list -- Thanks & Best Regards Hangbin Liu <liu...@gm...> |
From: <ch...@su...> - 2013-11-06 17:23:58
|
Hi! > new file mode 100644 > index 0000000..c698628 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c > @@ -0,0 +1,823 @@ > +/* > + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > + * > + * This program is free software; you can redistribute it and/or > + * modify it under the terms of the GNU General Public License as > + * published by the Free Software Foundation; either version 2 of > + * the License, or (at your option) any later version. > + * > + * This program is distributed in the hope that it would be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, write the Free Software Foundation, > + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + * > + * Author: Alexey Kodanev <ale...@or...> > + * > + */ > + > +#include <sys/stat.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <sys/time.h> > +#include <sys/resource.h> > +#include <sys/sysinfo.h> > +#include <sys/poll.h> > +#include <netinet/in.h> > +#include <arpa/inet.h> > +#include <netdb.h> > +#include <signal.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <pthread.h> > +#include <unistd.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <unistd.h> > +#include <string.h> > +#include <errno.h> > + > +#include "test.h" > +#include "usctest.h" > +#include "safe_macros.h" > + > +char *TCID = "tcp_fastopen"; > + > +static const int max_msg_len = 1500; > + > +/* TCP server requiers */ > +#ifndef TCP_FASTOPEN > +#define TCP_FASTOPEN 23 > +#endif > + > +/* TCP client requiers */ > +#ifndef MSG_FASTOPEN > +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ > +#endif > + > +enum { > + TCP_SERVER = 0, > + TCP_CLIENT, > +}; > +static int tcp_mode; > + > +enum { > + TFO_ENABLED = 0, > + TFO_DISABLED, > +}; > +static int tfo_support; > + > +enum { > + TCP_NEW_API = 0, > + TCP_OLD_API > +}; > +static int tcp_api; > + > +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; > +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; > +static int tw_reuse_changed; > +static int tfo_cfg_value; > +static int tfo_bit_num; > +static int tfo_cfg_changed; > + > +static int sfd; > + > +static int tfo_queue_size = 100; > +static int max_queue_len = 100; > +static int client_byte = 1; > +static int server_byte = 2; > +static const int start_byte = 0x24; > +static const int end_byte = 0x0a; > +static int client_msg_size = 32; > +static int server_msg_size = 128; > +static char *client_msg; > +static char *server_msg; > + > +/* > + * The number of requests from client after > + * which server has to close the connection. > + */ > +static int server_max_requests = 3; > +static int client_max_requests = 10; > +static int clients_num = 2; > +static char *tcp_port = "61000"; > +static char *server_addr = "localhost"; > +static char *rpath = "./tfo_result"; > + > +static int main_pid; > + > +socklen_t sock_len, csock_len; > + > +static int force_run; > + > +/* test options */ > +static char *narg, *Narg, *qarg, *barg, *Barg, > + *rarg, *Rarg, *aarg, *Targ; > +static int nflag, Nflag, qflag, bflag, Bflag, dflag, > + rflag, Rflag, aflag, Hflag, Tflag, gflag; There is no reason to use these flags. The option pointers are set to NULL (global variables). So that you can use: if (arg) { ... } In the check_opt() functions. > + {"o", &tcp_api, NULL}, This should be rather named as use_fastopen beacuse as it is, the name is misleading (I was wondering if there were some changes in the TCP userspace API for a while). > +static struct tcp_func tcp; > + > +#define MAX_THREADS 10000 > + > +/* current thread size */ > +static pthread_attr_t attr; > +static pthread_t *thread_ids; > +static int threads_num; > +static struct addrinfo *remote_addrinfo; > +static struct addrinfo *local_addrinfo; > +static const struct linger clo = { 1, 3 }; > + > +static void cleanup(void) > +{ > + static int first = 1; > + if (!first) > + return; > + first = 0; > + > + tst_resm(TINFO, "cleanup"); > + > + free(client_msg); > + free(server_msg); > + > + if (skip_cleanup) > + return; > + > + tcp.cleanup(); > + > + if (tfo_cfg_changed) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tfo_cfg_value &= ~tfo_bit_num; > + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); > + tst_resm(TINFO, "unset '%s' back to '%d'", > + tfo_cfg, tfo_cfg_value); > + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); > + } > + > + if (tw_reuse_changed) { > + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); > + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); > + } > + TEST_CLEANUP; > +} > + > +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) > +{ > + struct pollfd pfd; > + pfd.fd = *fd; > + pfd.events = POLLIN; > + int len = -1; > + while (1) { > + errno = 0; > + int ret = poll(&pfd, 1, wait_timeout / 1000); > + if (ret == -1) { > + if (errno == EINTR) > + continue; > + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", > + __FILE__, __LINE__); > + break; > + } > + if (ret == 0) { > + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); > + break; > + } > + > + if (ret != 1 || !(pfd.revents & POLLIN)) > + break; > + > + errno = 0; > + len = recv(*fd, buf + *offset, > + *buf_size - *offset, MSG_DONTWAIT); > + > + if (len == -1 && (errno == EINTR || > + errno == EWOULDBLOCK || errno == EAGAIN)) > + continue; > + else > + break; > + } > + > + if (len == 0) > + tst_resm(TINFO, "sock was closed '%d'", *fd); > + > + return len; > +} Now this one seems wrong. Both client and server creates new thread for each connection, which means that this creates bussy loop (which will eventually degrade performance). Why don't you use blocking read() so that each thread waits in read() syscall util data are ready? > +void client_recv(int *fd, char *buf) > +{ > + int len, offset = 0; > + > + while (1) { > + > + len = sock_recv_poll(fd, buf, &server_msg_size, &offset); > + > + /* socket closed or msg is not valid */ > + if (len < 1 || (offset + len) > server_msg_size || > + (buf[0] != start_byte && buf[0] != start_byte + 1)) { > + tst_resm(TFAIL | TERRNO, > + "recv failed, sock '%d'", *fd); > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) > + continue; > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *fd); > + } > + > + if (buf[0] == start_byte + 1) { > + /* recv last msg, close socket */ > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + } > + break; > + } > +} > + > +int client_connect_send(int *cfd) > +{ > + *cfd = socket(AF_INET, SOCK_STREAM, 0); > + const int flag = 1; > + setsockopt(*cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (*cfd == -1) { > + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", > + __FILE__, __LINE__); > + return -1; > + } > + > + if (tcp_api == TCP_NEW_API) { > + /* Replaces connect() + send()/write() */ > + if (sendto(*cfd, client_msg, client_msg_size, > + MSG_FASTOPEN | MSG_NOSIGNAL, > + remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "sendto failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } else { > + /* old TCP API */ > + if (connect(*cfd, remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen)) { > + tst_resm(TFAIL | TERRNO, "connect failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + > + if (send(*cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, > + "send failed on sock '%d'", *cfd); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } > + > + return 0; > +} > + > +void *client_fn(void *arg) > +{ > + char buf[server_msg_size]; > + int cfd, i; > + > + /* connect & send requests */ > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + for (i = 1; i < client_max_requests; ++i) { > + > + /* check connection, it can be closed */ > + int ret = 0; > + if (cfd != -1) > + ret = recv(cfd, buf, 1, MSG_DONTWAIT); > + > + if (ret == 0) { > + /* try to reconnect and send */ > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + continue; > + > + } else if (ret > 0) { > + tst_resm_hexd(TFAIL, buf, 1, > + "received after recv, sock '%d':", cfd); > + } > + > + if (verbose) { > + tst_resm_hexd(TINFO, client_msg, client_msg_size, > + "try to send msg[%d]", i); > + } > + > + if (send(cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "send failed"); > + break; > + } > + client_recv(&cfd, buf); > + } > + > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + return NULL; > +} > + > +union un_uint16 { > + char b[2]; > + uint16_t val; > +}; > + > +static struct timeval tv_client_start; > +static struct timeval tv_client_end; > + > +static void client_init(void) > +{ > + if (clients_num >= MAX_THREADS) { > + tst_brkm(TBROK, cleanup, > + "Unexpected num of clients '%d'", > + clients_num); > + } > + > + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); > + > + client_msg = SAFE_MALLOC(NULL, client_msg_size); > + memset(client_msg, client_byte, client_msg_size); > + client_msg[0] = start_byte; > + client_msg[1] = server_byte; > + > + union un_uint16 un; > + un.val = htons(server_msg_size); > + memcpy(client_msg + 2, un.b, 2); > + client_msg[client_msg_size - 1] = end_byte; > + > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + gettimeofday(&tv_client_start, NULL); > + int i; > + for (i = 0; i < clients_num; ++i) { > + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", > + __FILE__, __LINE__); > + } > + ++threads_num; > + } > +} > + > +static void client_run(void) > +{ > + void *res = NULL; > + long clnt_time = 0; > + > + int i; > + for (i = 0; i < clients_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + threads_num = 0; > + > + gettimeofday(&tv_client_end, NULL); > + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + > + tv_client_end.tv_usec - tv_client_start.tv_usec; > + > + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); > + > + /* ask server to terminate */ > + int cfd; > + client_msg[0] = start_byte + 1; > + if (!client_connect_send(&cfd)) { > + shutdown(cfd, SHUT_WR); > + SAFE_CLOSE(NULL, cfd); > + } > + > + /* the script tcp_fastopen_run.sh will remove it */ > + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); > +} > + > +static void client_cleanup(void) > +{ > + void *res = NULL; > + int i; > + for (i = 0; i < threads_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + free(thread_ids); > + > + if (remote_addrinfo) > + freeaddrinfo(remote_addrinfo); > +} > + > +void make_server_reply(const char *buf, char **msg, uint16_t *size) > +{ > + union un_uint16 un; > + memcpy(un.b, buf + 2, 2); > + *size = ntohs(un.val); > + if (*size < 2 || *size > max_msg_len) > + *size = 2; > + > + if (msg == NULL) { > + tst_resm(TWARN, "failed to make a reply"); > + return; > + } > + > + *msg = SAFE_MALLOC(NULL, *size); > + memset(*msg, buf[1], *size - 1); > + (*msg)[*size - 1] = end_byte; > +} This part of the code is really cryptic, the un.val is magically initialized via the memset from buf and the whole code does not make much sense at first sight. I've been looking at the code for more than ten minutes but I can't figure out what exactly does this do. I suppose that the size of the message is constructed from the data received by the client and that reply message buffer is allocated accordingly. Can you please write the code in a way that it's clear what it does? (i.e. parse the request, store the data into well named variables and then construct the reply message accordingly). > +void *server_fn(void *cfd) > +{ > + int *client_fd = cfd; > + int num_requests = 0, offset = 0; > + uint16_t reply_size = 0; > + char *msg = NULL, buf[max_msg_len]; > + setsockopt(*client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); > + ssize_t len; > + > + while (1) { > + errno = 0; > + len = sock_recv_poll(client_fd, buf, &max_msg_len, &offset); > + > + if (len == 0) { > + break; > + } else if (len < 0 || (offset + len) > max_msg_len || > + (buf[0] != start_byte && buf[0] != start_byte + 1)) { > + tst_resm(TFAIL, "recv failed, sock '%d'", *client_fd); > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) { > + tst_resm(TINFO, "msg is not complete, continue recv"); > + continue; > + } > + > + if (buf[0] == start_byte + 1) > + tst_brkm(TBROK, cleanup, "client asks to terminate..."); > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *client_fd); > + } > + > + if (msg == NULL) > + make_server_reply(buf, &msg, &reply_size); > + > + ++num_requests; > + > + /* if last message */ > + if (num_requests >= server_max_requests) > + msg[0] = start_byte + 1; > + else > + msg[0] = start_byte; > + > + if (send(*client_fd, msg, reply_size, MSG_NOSIGNAL) == -1) > + tst_resm(TWARN | TERRNO, "Error while sending msg"); > + else { > + if (verbose) { > + tst_resm_hexd(TINFO, msg, reply_size, > + "msg sent:"); > + } > + } > + > + offset = 0; > + > + if (num_requests >= server_max_requests) { > + if (verbose) > + tst_resm(TINFO, "Max reqs, close socket"); > + shutdown(*client_fd, SHUT_WR); > + break; > + } > + } > + > + free(msg); > + SAFE_CLOSE(NULL, *client_fd); > + free(client_fd); > + > + return NULL; > +} > + > +static void server_thread_add(int *client_fd) > +{ > + int *cfd = SAFE_MALLOC(NULL, sizeof(int)); > + *cfd = *client_fd; > + pthread_t id; You don't have to allocate the cfd here, just cast the client fd (not a pointer to it, the value) to intptr_t and pass it as arg. Then in the server_fd cast the arg back to int and assign the value to int variable. > + if (pthread_create(&id, &attr, server_fn, cfd) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", __FILE__, __LINE__); > + } > +} > + > +static void server_init(void) > +{ > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + hints.ai_flags = AI_PASSIVE; > + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + sfd = socket(AF_INET, SOCK_STREAM, 0); > + if (sfd == -1) > + tst_brkm(TBROK, cleanup, "Failed to create a socket"); > + > + tst_resm(TINFO, "assigning a name to the server socket..."); > + if (!local_addrinfo) > + tst_brkm(TBROK, cleanup, "failed to get the address"); > + > + while (bind(sfd, local_addrinfo->ai_addr, > + local_addrinfo->ai_addrlen) == -1) { > + sleep(1); > + } > + tst_resm(TINFO, "the name assigned"); > + > + const int flag = 1; > + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (tcp_api == TCP_NEW_API) { > + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, > + sizeof(tfo_queue_size)) == -1) > + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); > + } > + > + listen(sfd, max_queue_len); > + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); > +} > + > +static void server_cleanup(void) > +{ > + SAFE_CLOSE(NULL, sfd); > + if (local_addrinfo) > + freeaddrinfo(local_addrinfo); > +} > + > +static void server_run(void) > +{ > + struct sockaddr_in client_addr; > + socklen_t addr_size = sizeof(client_addr); > + pthread_attr_init(&attr); > + > + /* > + * detaching threads allow to reclaim thread's resources > + * ones a thread finishs its work. ^ Should be 'once' > + */ > + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "setdetached failed"); > + > + while (1) { > + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, > + &addr_size); > + if (client_fd == -1) { > + tst_brkm(TBROK, cleanup, "Can't create client socket"); > + continue; > + } > + > + if (client_addr.sin_family == AF_INET) { > + if (verbose) { > + tst_resm(TINFO, "conn: port '%d', addr '%s'", > + client_addr.sin_port, > + inet_ntoa(client_addr.sin_addr)); > + } > + } > + server_thread_add(&client_fd); > + } > +} > + > +static void check_opt(const char *name, int *flag, > + char *arg, int *val, int lim) > +{ > + if (*flag) { > + if (sscanf(arg, "%i", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", > + name, lim); > + } > +} > + > +static void check_opt_l(const char *name, int *flag, > + char *arg, long *val, long lim) > +{ > + if (*flag) { > + if (sscanf(arg, "%ld", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", > + name, lim); > + } > +} > + > +/* cleanup flags */ > +void setup(int argc, char *argv[]) > +{ > + char *msg; > + msg = parse_opts(argc, argv, options, help); > + if (msg != NULL) > + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); > + > + /* if client num is not set, use num of processors */ > + struct sysinfo si; > + if (sysinfo(&si) == 0) > + clients_num = si.procs; > + > + check_opt("a", &aflag, aarg, &clients_num, 1); > + check_opt("r", &rflag, rarg, &client_max_requests, 1); > + check_opt("R", &Rflag, Rarg, &server_max_requests, 1); > + check_opt("n", &nflag, narg, &client_msg_size, 1); > + > + check_opt("b", &bflag, barg, &client_byte, 0); > + if (client_byte == start_byte || client_byte == end_byte) > + tst_brkm(TBROK, NULL, "-b option, wrong arg"); > + > + check_opt("N", &Nflag, Narg, &server_msg_size, 1); > + > + check_opt("B", &Bflag, Barg, &server_byte, 0); > + if (server_byte == start_byte || server_byte == end_byte) > + tst_brkm(TBROK, NULL, "-B option, wrong arg"); > + > + check_opt("q", &qflag, qarg, &tfo_queue_size, 1); > + check_opt_l("T", &Tflag, Targ, &wait_timeout, 0L); > + > + if (!force_run) > + tst_require_root(NULL); > + > + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { > + tst_brkm(TCONF, NULL, > + "Test must be run with kernel 3.7 or newer"); > + } > + > + /* check tcp fast open knob */ > + if (!force_run && access(tfo_cfg, F_OK) == -1) > + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); > + > + if (!force_run) { > + SAFE_FILE_SCANF(cleanup, tfo_cfg, "%d", &tfo_cfg_value); > + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); > + } > + > + tst_sig(FORK, DEF_HANDLER, cleanup); > + > + main_pid = getpid(); > + tst_resm(TINFO, "pid '%d'", main_pid); > + tst_resm(TINFO, "TCP %s is using %s TCP API.", > + (tcp_mode == TCP_SERVER) ? "server" : "client", > + (tcp_api == TCP_NEW_API) ? "new" : "old"); > + > + switch (tcp_mode) { > + case TCP_SERVER: > + tst_resm(TINFO, "max requests '%d'", > + server_max_requests); > + tcp.init = server_init; > + tcp.run = server_run; > + tcp.cleanup = server_cleanup; > + tfo_bit_num = 2; > + break; > + case TCP_CLIENT: > + tst_resm(TINFO, "connection: %s:%s", > + server_addr, tcp_port); > + tst_resm(TINFO, "client max req: %d", client_max_requests); > + tst_resm(TINFO, "clients num: %d", clients_num); > + tst_resm(TINFO, "client msg size: %d", client_msg_size); > + tst_resm(TINFO, "server msg size: %d", server_msg_size); > + tst_resm(TINFO, "client msg byte: %02x", client_byte); > + tst_resm(TINFO, "server msg byte: %02x", server_byte); > + > + tcp.init = client_init; > + tcp.run = client_run; > + tcp.cleanup = client_cleanup; > + tfo_bit_num = 1; > + break; > + } > + > + tfo_support = TFO_ENABLED == tfo_support; > + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { > + int value = (tfo_cfg_value & ~tfo_bit_num) > + | (tfo_support << (tfo_bit_num - 1)); > + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); > + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); > + tfo_cfg_changed = 1; > + } > + > + int reuse_value = 0; > + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); > + if (!reuse_value) { > + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); > + tw_reuse_changed = 1; > + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); > + } > + > + tst_resm(TINFO, "TFO support %s", > + (tfo_support) ? "enabled" : "disabled"); > + > + tcp.init(); > +} > + > +static void test_run() > +{ > + tcp.run(); > +} Just call tcp.run() in the main() this is useless redirection. > +int main(int argc, char *argv[]) > +{ > + setup(argc, argv); > + > + test_run(); > + > + cleanup(); > + > + tst_exit(); > +} > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > new file mode 100755 > index 0000000..d56c510 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > @@ -0,0 +1,190 @@ > +#!/bin/bash > + > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > +# > +# Author: Alexey Kodanev <ale...@or...> > +# > + > +# default command-line options > +user_name="root" > +remote_addr=$RHOST > +use_ssh=0 > +client_requests=2000000 > +server_port=$[ $RANDOM % 28232 + 32768 ] > + > +while getopts :hu:H:sr:p: opt; do > + case "$opt" in > + h) > + echo "Usage:" > + echo "h help" > + echo "u x server user name" > + echo "H x server hostname or IP address" > + echo "s use ssh to run remote cmds" > + echo "r x the number of requests" > + echo "p x server port" > + exit 0 > + ;; > + u) user_name=$OPTARG ;; > + H) remote_addr=$OPTARG ;; > + s) use_ssh=1 ;; > + r) client_requests=$OPTARG ;; > + p) server_port=$OPTARG ;; > + *) > + tst_brkm TBROK NULL "unknown option: $opt" > + exit 2 > + ;; > + esac > +done > + > +run_remote_cmd() > +{ > + tst_resm TINFO "run cmd on $remote_addr: $1" > + > + if [ "$use_ssh" = 1 ]; then > + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" > + else > + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" > + fi > +} > + > +cleanup() > +{ > + rm -f $tfo_result > + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" > + # remove test files on remote host > + sleep 2 > + run_remote_cmd "[ -e ${tdir}tcp_fastopen ] && rm -f ${tdir}tcp_fastopen" > + sleep 2 > +} > + > +read_result_file() > +{ > + if [ -f $tfo_result ]; then > + if [ -r $tfo_result ]; then > + cat $tfo_result > + else > + tst_brkm TBROK NULL "Failed to read result file" > + exit 2 > + fi > + else > + tst_brkm TBROK NULL "Failed to find result file" > + exit 2 > + fi > +} > + > +check_exit_status() > +{ > + if [ "$1" -ne "0" ]; then > + tst_brkm TBROK NULL "Last test has failed" > + exit $1; > + fi > +} > + > +export RC=0 > +export TST_TOTAL=1 > +export TCID="tcp_fastopen" > +export TST_COUNT=0 > + > +tfo_result_ms=0 > +bind_timeout=10 > +clients_num=2 > +max_requests=3 > +msg_timeout=10000000 > + > +# Setup > +type tst_resm > /dev/null 2>&1 > +if [ $? -eq 1 ]; then > + echo "$TCID 0 TCONF : failed to find LTP tst_* utilities" > + exit 2 > +fi There is no need to check for tst_ commands. If these are not in $PATH none of the LTP shell scripts will work. > +tst_kvercmp 3 7 0 > +if [ $? -eq 0 ]; then > + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" > + exit 0 > +fi > + > +if [ -z $remote_addr ]; then > + tst_brkm TBROK NULL "you must specify server address" > + exit 2 > +fi Can we default to localhost, would that work? > +if [ -z $LTPROOT ]; then > + tdir="./" > +else > + tdir=${LTPROOT}/testcases/bin/ > +fi This should not be used, the ${LTPROOT}/testcases/bin/ must be in $PATH prior to execution of the test. > +if [ -z "$TMPDIR" ]; then > + tfo_result="${tdir}tfo_result" > +else > + tfo_result="${TMPDIR}/tfo_result" > +fi As well as TMPDIR. > +trap "cleanup" EXIT > + > +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" > + > +if [ "$use_ssh" = 1 ]; then > + scp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null > +else > + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > /dev/null > +fi > + > + > +# Run > +tcp_api_opt=("-o -O" "") > +vtime=(0 0) > + > +for (( i = 0; i < 2; ++i )); do > + # kill tcp server on remote machine > + run_remote_cmd "pgrep tcp_fastopen && killall -9 tcp_fastopen" > + > + sleep 2 > + > + # run tcp server on remote machine > + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests -q 100 \ > +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" > + > + sleep $bind_timeout > + > + # run local tcp client > + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ > +-n 32 -b 23 -N 128 -B 8 -T $msg_timeout -H $remote_addr \ > +${tcp_api_opt[$i]} -g $server_port -d $tfo_result > + > + check_exit_status $? > + > + vtime[$i]=`read_result_file` > + > + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then > + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" > + exit 2 > + fi > + server_port=$[ $server_port + 1 ] > +done > + > +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] > + > +if (( $tfo_cmp < 3 )); then > + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" > + exit 1 > +fi > + > +tst_resm TPASS "TFO performance result is $tfo_cmp percent" > +exit 0 -- Cyril Hrubis ch...@su... |
From: <ale...@or...> - 2013-11-07 09:33:34
|
Hi! On 11/06/2013 07:05 PM, ch...@su... wrote: > Hi! >> + >> +/* test options */ >> +static char *narg, *Narg, *qarg, *barg, *Barg, >> + *rarg, *Rarg, *aarg, *Targ; >> +static int nflag, Nflag, qflag, bflag, Bflag, dflag, >> + rflag, Rflag, aflag, Hflag, Tflag, gflag; > There is no reason to use these flags. The option pointers are set to > NULL (global variables). So that you can use: > > if (arg) { > ... > } > > In the check_opt() functions. OK >> + {"o", &tcp_api, NULL}, > This should be rather named as use_fastopen beacuse as it is, the name > is misleading (I was wondering if there were some changes in the TCP > userspace API for a while). There are changes in user-space TCP API: we don't use normal TCP system calls: connect(), send(), write(). Instead making use of sendmsg(), sendto() which are normally used in UDP. Some additional flags were introduced. > ... >> +static struct tcp_func tcp; >> + >> +#define MAX_THREADS 10000 >> + >> +/* current thread size */ >> +static pthread_attr_t attr; >> +static pthread_t *thread_ids; >> +static int threads_num; >> +static struct addrinfo *remote_addrinfo; >> +static struct addrinfo *local_addrinfo; >> +static const struct linger clo = { 1, 3 }; >> + >> +static void cleanup(void) >> +{ >> + static int first = 1; >> + if (!first) >> + return; >> + first = 0; >> + >> + tst_resm(TINFO, "cleanup"); >> + >> + free(client_msg); >> + free(server_msg); >> + >> + if (skip_cleanup) >> + return; >> + >> + tcp.cleanup(); >> + >> + if (tfo_cfg_changed) { >> + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); >> + tfo_cfg_value &= ~tfo_bit_num; >> + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); >> + tst_resm(TINFO, "unset '%s' back to '%d'", >> + tfo_cfg, tfo_cfg_value); >> + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); >> + } >> + >> + if (tw_reuse_changed) { >> + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); >> + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); >> + } >> + TEST_CLEANUP; >> +} >> + >> +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) >> +{ >> + struct pollfd pfd; >> + pfd.fd = *fd; >> + pfd.events = POLLIN; >> + int len = -1; >> + while (1) { >> + errno = 0; >> + int ret = poll(&pfd, 1, wait_timeout / 1000); >> + if (ret == -1) { >> + if (errno == EINTR) >> + continue; >> + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", >> + __FILE__, __LINE__); >> + break; >> + } >> + if (ret == 0) { >> + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); >> + break; >> + } >> + >> + if (ret != 1 || !(pfd.revents & POLLIN)) >> + break; >> + >> + errno = 0; >> + len = recv(*fd, buf + *offset, >> + *buf_size - *offset, MSG_DONTWAIT); >> + >> + if (len == -1 && (errno == EINTR || >> + errno == EWOULDBLOCK || errno == EAGAIN)) >> + continue; >> + else >> + break; >> + } >> + >> + if (len == 0) >> + tst_resm(TINFO, "sock was closed '%d'", *fd); >> + >> + return len; >> +} > Now this one seems wrong. Both client and server creates new thread for > each connection, which means that this creates bussy loop (which will > eventually degrade performance). Why don't you use blocking read() so > that each thread waits in read() syscall util data are ready? It's not busy loop here, poll() will block. The purpose of the while(1) is to go once more if the poll call was interrupted. I think I need to remove EWOULDBLOCK and EGAIN errors checking as poll doesn't have their. I didn't use read with block because I wanted some sort of block wait timeout, which poll has. > ... >> + >> +static void client_init(void) >> +{ >> + if (clients_num >= MAX_THREADS) { >> + tst_brkm(TBROK, cleanup, >> + "Unexpected num of clients '%d'", >> + clients_num); >> + } >> + >> + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); >> + >> + client_msg = SAFE_MALLOC(NULL, client_msg_size); >> + memset(client_msg, client_byte, client_msg_size); >> + client_msg[0] = start_byte; >> + client_msg[1] = server_byte; >> + >> + union un_uint16 un; >> + un.val = htons(server_msg_size); >> + memcpy(client_msg + 2, un.b, 2); >> + client_msg[client_msg_size - 1] = end_byte; >> + >> + struct addrinfo hints; >> + memset(&hints, 0, sizeof(struct addrinfo)); >> + hints.ai_family = AF_INET; >> + hints.ai_socktype = SOCK_STREAM; >> + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) >> + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); >> + >> + gettimeofday(&tv_client_start, NULL); >> + int i; >> + for (i = 0; i < clients_num; ++i) { >> + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { >> + tst_brkm(TBROK | TERRNO, cleanup, >> + "pthread_create failed at %s:%d", >> + __FILE__, __LINE__); >> + } >> + ++threads_num; >> + } >> +} >> + >> +static void client_run(void) >> +{ >> + void *res = NULL; >> + long clnt_time = 0; >> + >> + int i; >> + for (i = 0; i < clients_num; ++i) >> + pthread_join(thread_ids[i], &res); >> + >> + threads_num = 0; >> + >> + gettimeofday(&tv_client_end, NULL); >> + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + >> + tv_client_end.tv_usec - tv_client_start.tv_usec; >> + >> + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); >> + >> + /* ask server to terminate */ >> + int cfd; >> + client_msg[0] = start_byte + 1; >> + if (!client_connect_send(&cfd)) { >> + shutdown(cfd, SHUT_WR); >> + SAFE_CLOSE(NULL, cfd); >> + } >> + >> + /* the script tcp_fastopen_run.sh will remove it */ >> + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); >> +} >> + >> +static void client_cleanup(void) >> +{ >> + void *res = NULL; >> + int i; >> + for (i = 0; i < threads_num; ++i) >> + pthread_join(thread_ids[i], &res); >> + >> + free(thread_ids); >> + >> + if (remote_addrinfo) >> + freeaddrinfo(remote_addrinfo); >> +} >> + >> +void make_server_reply(const char *buf, char **msg, uint16_t *size) >> +{ >> + union un_uint16 un; >> + memcpy(un.b, buf + 2, 2); >> + *size = ntohs(un.val); >> + if (*size < 2 || *size > max_msg_len) >> + *size = 2; >> + >> + if (msg == NULL) { >> + tst_resm(TWARN, "failed to make a reply"); >> + return; >> + } >> + >> + *msg = SAFE_MALLOC(NULL, *size); >> + memset(*msg, buf[1], *size - 1); >> + (*msg)[*size - 1] = end_byte; >> +} > This part of the code is really cryptic, the un.val is magically > initialized via the memset from buf and the whole code does not make > much sense at first sight. I've been looking at the code for more than > ten minutes but I can't figure out what exactly does this do. > > I suppose that the size of the message is constructed from the data > received by the client and that reply message buffer is allocated > accordingly. Can you please write the code in a way that it's clear what > it does? (i.e. parse the request, store the data into well named > variables and then construct the reply message accordingly). Yes, you are right. I will rewrite it >> +void *server_fn(void *cfd) >> +{ >> + int *client_fd = cfd; >> + int num_requests = 0, offset = 0; >> + uint16_t reply_size = 0; >> + char *msg = NULL, buf[max_msg_len]; >> + setsockopt(*client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); >> + ssize_t len; >> + >> + while (1) { >> + errno = 0; >> + len = sock_recv_poll(client_fd, buf, &max_msg_len, &offset); >> + >> + if (len == 0) { >> + break; >> + } else if (len < 0 || (offset + len) > max_msg_len || >> + (buf[0] != start_byte && buf[0] != start_byte + 1)) { >> + tst_resm(TFAIL, "recv failed, sock '%d'", *client_fd); >> + break; >> + } >> + >> + offset += len; >> + >> + if (buf[offset - 1] != end_byte) { >> + tst_resm(TINFO, "msg is not complete, continue recv"); >> + continue; >> + } >> + >> + if (buf[0] == start_byte + 1) >> + tst_brkm(TBROK, cleanup, "client asks to terminate..."); >> + >> + if (verbose) { >> + tst_resm_hexd(TINFO, buf, offset, >> + "msg recv from sock %d:", *client_fd); >> + } >> + >> + if (msg == NULL) >> + make_server_reply(buf, &msg, &reply_size); >> + >> + ++num_requests; >> + >> + /* if last message */ >> + if (num_requests >= server_max_requests) >> + msg[0] = start_byte + 1; >> + else >> + msg[0] = start_byte; >> + >> + if (send(*client_fd, msg, reply_size, MSG_NOSIGNAL) == -1) >> + tst_resm(TWARN | TERRNO, "Error while sending msg"); >> + else { >> + if (verbose) { >> + tst_resm_hexd(TINFO, msg, reply_size, >> + "msg sent:"); >> + } >> + } >> + >> + offset = 0; >> + >> + if (num_requests >= server_max_requests) { >> + if (verbose) >> + tst_resm(TINFO, "Max reqs, close socket"); >> + shutdown(*client_fd, SHUT_WR); >> + break; >> + } >> + } >> + >> + free(msg); >> + SAFE_CLOSE(NULL, *client_fd); >> + free(client_fd); >> + >> + return NULL; >> +} >> + >> +static void server_thread_add(int *client_fd) >> +{ >> + int *cfd = SAFE_MALLOC(NULL, sizeof(int)); >> + *cfd = *client_fd; >> + pthread_t id; > You don't have to allocate the cfd here, just cast the client fd (not a > pointer to it, the value) to intptr_t and pass it as arg. Then in the > server_fd cast the arg back to int and assign the value to int variable. Great! I will do that way. > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > new file mode 100755 > index 0000000..d56c510 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > @@ -0,0 +1,190 @@ > +#!/bin/bash > ... > + > +# Setup > +type tst_resm > /dev/null 2>&1 > +if [ $? -eq 1 ]; then > + echo "$TCID 0 TCONF : failed to find LTP tst_* utilities" > + exit 2 > +fi > There is no need to check for tst_ commands. If these are not in $PATH > none of the LTP shell scripts will work. OK >> +tst_kvercmp 3 7 0 >> +if [ $? -eq 0 ]; then >> + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" >> + exit 0 >> +fi >> + >> +if [ -z $remote_addr ]; then >> + tst_brkm TBROK NULL "you must specify server address" >> + exit 2 >> +fi > Can we default to localhost, would that work? It will work, but we won't see some significant performance results. |
From: <ch...@su...> - 2013-11-07 10:07:18
|
Hi! > >> + > >> +/* test options */ > >> +static char *narg, *Narg, *qarg, *barg, *Barg, > >> + *rarg, *Rarg, *aarg, *Targ; > >> +static int nflag, Nflag, qflag, bflag, Bflag, dflag, > >> + rflag, Rflag, aflag, Hflag, Tflag, gflag; > > There is no reason to use these flags. The option pointers are set to > > NULL (global variables). So that you can use: > > > > if (arg) { > > ... > > } > > > > In the check_opt() functions. > OK > >> + {"o", &tcp_api, NULL}, > > This should be rather named as use_fastopen beacuse as it is, the name > > is misleading (I was wondering if there were some changes in the TCP > > userspace API for a while). > There are changes in user-space TCP API: we don't use normal TCP system > calls: connect(), send(), write(). Instead making use of sendmsg(), > sendto() which are normally used in UDP. Some additional flags were > introduced. Still I think it would be better to use 'fastopen' string in the opt name. > >> +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) > >> +{ > >> + struct pollfd pfd; > >> + pfd.fd = *fd; > >> + pfd.events = POLLIN; > >> + int len = -1; > >> + while (1) { > >> + errno = 0; > >> + int ret = poll(&pfd, 1, wait_timeout / 1000); > >> + if (ret == -1) { > >> + if (errno == EINTR) > >> + continue; > >> + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", > >> + __FILE__, __LINE__); > >> + break; > >> + } > >> + if (ret == 0) { > >> + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); > >> + break; > >> + } > >> + > >> + if (ret != 1 || !(pfd.revents & POLLIN)) > >> + break; > >> + > >> + errno = 0; > >> + len = recv(*fd, buf + *offset, > >> + *buf_size - *offset, MSG_DONTWAIT); > >> + > >> + if (len == -1 && (errno == EINTR || > >> + errno == EWOULDBLOCK || errno == EAGAIN)) > >> + continue; > >> + else > >> + break; > >> + } > >> + > >> + if (len == 0) > >> + tst_resm(TINFO, "sock was closed '%d'", *fd); > >> + > >> + return len; > >> +} > > Now this one seems wrong. Both client and server creates new thread for > > each connection, which means that this creates bussy loop (which will > > eventually degrade performance). Why don't you use blocking read() so > > that each thread waits in read() syscall util data are ready? > It's not busy loop here, poll() will block. The purpose of the while(1) > is to go once more if the poll call was interrupted. I think I need to > remove EWOULDBLOCK and EGAIN errors checking as poll doesn't have their. > I didn't use read with block because I wanted some sort of block wait > timeout, which poll has. Ah my bad, I see it now. > > Can we default to localhost, would that work? > It will work, but we won't see some significant performance results. Ok. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2013-11-11 14:41:47
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/README | 16 + testcases/network/tcp_fastopen/tcp_fastopen.c | 835 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 173 ++++ 5 files changed, 1049 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/README create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..bff25e5 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/README b/testcases/network/tcp_fastopen/README new file mode 100644 index 0000000..4e19add --- /dev/null +++ b/testcases/network/tcp_fastopen/README @@ -0,0 +1,16 @@ +TCP Fast Open (TFO) is an extension to speed up the opening of TCP connections +between two endpoints. It reduces the number of round time trips (RTT) +required in TCP conversations. + +The test must be run on Linux versions higher then 3.7. (client side +implemented in Linux 3.6, server side in Linux 3.7). + +Developers reported (in "TCP Fast Open", CoNEXT 2011), TFO could result in +speed improvements of between 4% and 41% in the page load times on popular web +sites. + +With the default test scenario, a client is constantly sending requests to +a server and waiting for replies. The server is closing a connection after 3 +requests from the client. This scenario was chosen to simulate an average +conversation between a web-browser and an application server. So the test +results with TFO support enabled must be at least 3 percent faster. diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..5e62781 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,835 @@ +/* + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/sysinfo.h> +#include <sys/poll.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static int client_byte = 0x43; +static int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 2000000; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *barg, *Barg, + *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"b:", NULL, &barg}, + {"N:", NULL, &Narg}, + {"B:", NULL, &Barg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -b x x is a byte of client message\n"); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -B x x is a byte of server message\n"); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; +static int threads_num; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void cleanup(void) +{ + static int first = 1; + if (!first) + return; + first = 0; + + tst_resm(TINFO, "cleanup"); + + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} + +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = *fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", + __FILE__, __LINE__); + break; + } + if (ret == 0) { + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(*fd, buf + *offset, + *buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + if (len == 0) + tst_resm(TINFO, "sock was closed '%d'", *fd); + + return len; +} + +void client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + + len = sock_recv_poll(fd, buf, &server_msg_size, &offset); + + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + tst_resm(TFAIL | TERRNO, + "recv failed, sock '%d'", *fd); + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + break; + } + + offset += len; + + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + if (buf[0] == start_fin_byte) { + /* recv last msg, close socket */ + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + } + break; + } +} + +int client_connect_send(int *cfd) +{ + *cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(*cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (*cfd == -1) { + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", + __FILE__, __LINE__); + return -1; + } + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(*cfd, client_msg, client_msg_size, + MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != client_msg_size) { + tst_resm(TFAIL | TERRNO, "sendto failed"); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(*cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + tst_resm(TFAIL | TERRNO, "connect failed"); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + + if (send(*cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, + "send failed on sock '%d'", *cfd); + SAFE_CLOSE(NULL, *cfd); + return -1; + } + } + + return 0; +} + +void *client_fn(void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + + /* connect & send requests */ + if (client_connect_send(&cfd)) + return NULL; + client_recv(&cfd, buf); + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + + if (client_connect_send(&cfd)) + return NULL; + client_recv(&cfd, buf); + + continue; + + } else if (ret > 0) { + tst_resm_hexd(TFAIL, buf, 1, + "received after recv, sock '%d':", cfd); + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, "send failed"); + break; + } + client_recv(&cfd, buf); + } + + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + return NULL; +} + +union uint2bytes { + char byte[2]; + uint16_t val; +}; + +static struct timeval tv_client_start; +static struct timeval tv_client_end; + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* reply will be filled with this byte */ + client_msg[1] = server_byte; + + /* set size for reply */ + union uint2bytes reply_size; + reply_size.val = htons(server_msg_size); + + /* write requested reply size to bytes 2, 3 */ + client_msg[2] = reply_size.byte[0]; + client_msg[3] = reply_size.byte[1]; + + client_msg[client_msg_size - 1] = end_byte; +} + +void parse_client_request(const char *recv_msg, uint16_t *send_msg_size, + char *send_msg_byte) +{ + *send_msg_byte = recv_msg[1]; + + /* read requested reply size */ + union uint2bytes reply_size; + reply_size.byte[0] = recv_msg[2]; + reply_size.byte[1] = recv_msg[3]; + *send_msg_size = ntohs(reply_size.val); + if (*send_msg_size < 2 || *send_msg_size > max_msg_len) { + tst_resm(TWARN, "wrong msg size received %u", *send_msg_size); + *send_msg_size = 2; + } +} + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + gettimeofday(&tv_client_start, NULL); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + ++threads_num; + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + + int i; + for (i = 0; i < clients_num; ++i) + pthread_join(thread_ids[i], &res); + + threads_num = 0; + + gettimeofday(&tv_client_end, NULL); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + + tv_client_end.tv_usec - tv_client_start.tv_usec; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); + + /* ask server to terminate */ + int cfd; + client_msg[0] = start_fin_byte; + if (!client_connect_send(&cfd)) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); +} + +static void client_cleanup(void) +{ + void *res = NULL; + int i; + for (i = 0; i < threads_num; ++i) + pthread_join(thread_ids[i], &res); + + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + + +void make_server_reply(char **send_msg, const uint16_t *send_msg_size, + const char *send_msg_byte) +{ + *send_msg = SAFE_MALLOC(NULL, *send_msg_size); + memset(*send_msg, *send_msg_byte, *send_msg_size - 1); + + (*send_msg)[0] = start_byte; + + (*send_msg)[*send_msg_size - 1] = end_byte; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL, send_msg_byte; + uint16_t send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + errno = 0; + recv_len = sock_recv_poll(&client_fd, recv_msg, + &max_msg_len, &offset); + + if (recv_len == 0) { + break; + } else if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); + break; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + tst_resm(TINFO, "msg is not complete, continue recv"); + continue; + } + + if (recv_msg[0] == start_fin_byte) + tst_brkm(TBROK, cleanup, "client asks to terminate..."); + + if (verbose) { + tst_resm_hexd(TINFO, recv_msg, offset, + "msg recv from sock %d:", client_fd); + } + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + parse_client_request(recv_msg, &send_msg_size, + &send_msg_byte); + + make_server_reply(&send_msg, &send_msg_size, + &send_msg_byte); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) + tst_resm(TWARN | TERRNO, "Error while sending msg"); + else { + if (verbose) { + tst_resm_hexd(TINFO, send_msg, send_msg_size, + "msg sent:"); + } + } + + offset = 0; + + if (num_requests >= server_max_requests) { + if (verbose) + tst_resm(TINFO, "Max reqs, close socket"); + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + SAFE_CLOSE(NULL, client_fd); + + return NULL; +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (intptr_t *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + sleep(1); + } + tst_resm(TINFO, "the name assigned"); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); + if (local_addrinfo) + freeaddrinfo(local_addrinfo); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) { + tst_brkm(TBROK, cleanup, "Can't create client socket"); + continue; + } + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void check_msg_byte(int *byte) +{ + /* these bytes are reserved */ + if (*byte == start_byte || *byte == start_fin_byte || + *byte == end_byte) + tst_brkm(TBROK, NULL, + "-B option, 0x%02x is reserved", *byte); +} + +/* cleanup flags */ +void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client num is not set, use num of processors */ + struct sysinfo si; + if (sysinfo(&si) == 0) + clients_num = si.procs; + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + + check_opt("b", barg, &client_byte, 0); + check_msg_byte(&client_byte); + + check_opt("N", Narg, &server_msg_size, 1); + + check_opt("B", Barg, &server_byte, 0); + check_msg_byte(&server_byte); + + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + tst_resm(TINFO, "client msg byte: %02x", client_byte); + tst_resm(TINFO, "server msg byte: %02x", server_byte); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..ae77b58 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 +server_port=$[ $RANDOM % 28232 + 32768 ] + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + echo "p x server port" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + p) server_port=$OPTARG ;; + *) + tst_brkm TBROK NULL "unknown option: $opt" + exit 2 + ;; + esac +done + +run_remote_cmd() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + + if [ "$use_ssh" = 1 ]; then + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" + else + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" + fi +} + +cleanup() +{ + rm -f $tfo_result + run_remote_cmd "pkill -9 tcp_fastopen\$" +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK NULL "Failed to read result file" + exit 2 + fi + else + tst_brkm TBROK NULL "Failed to find result file" + exit 2 + fi +} + +check_exit_status() +{ + if [ "$1" -ne "0" ]; then + tst_brkm TBROK NULL "Last test has failed" + exit $1; + fi +} + +export RC=0 +export TST_TOTAL=1 +export TCID="tcp_fastopen" +export TST_COUNT=0 + +tfo_result_ms=0 +bind_timeout=30 + + +tst_kvercmp 3 7 0 +if [ $? -eq 0 ]; then + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" + exit 0 +fi + +if [ -z $remote_addr ]; then + tst_brkm TBROK NULL "you must specify server address" + exit 2 +fi + +tdir="${LTPROOT}/testcases/bin/" +tfo_result="${TMPDIR}/tfo_result" + +trap "cleanup" EXIT + +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" + +if [ "$use_ssh" = 1 ]; then + scp -q ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir +else + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir +fi + +# Run test without/with TFO +tcp_api_opt=("-o -O" "") + +# Time results for without/with TFO test +vtime=(0 0) + +for (( i = 0; i < 2; ++i )); do + # kill tcp server on remote machine + run_remote_cmd "pkill -9 tcp_fastopen\$" + + sleep 2 + + # run tcp server on remote machine + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" + + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr ${tcp_api_opt[$i]} -g $server_port -d $tfo_result + + check_exit_status $? + + vtime[$i]=`read_result_file` + + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" + exit 2 + fi + server_port=$[ $server_port + 1 ] +done + +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] + +if (( $tfo_cmp < 3 )); then + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" + exit 1 +fi + +tst_resm TPASS "TFO performance result is $tfo_cmp percent" +exit 0 -- 1.7.1 |
From: <ch...@su...> - 2013-11-25 18:41:05
|
Hi! > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c > new file mode 100644 > index 0000000..5e62781 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c > @@ -0,0 +1,835 @@ > +/* > + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > + * > + * This program is free software; you can redistribute it and/or > + * modify it under the terms of the GNU General Public License as > + * published by the Free Software Foundation; either version 2 of > + * the License, or (at your option) any later version. > + * > + * This program is distributed in the hope that it would be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, write the Free Software Foundation, > + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + * > + * Author: Alexey Kodanev <ale...@or...> > + * > + */ > + > +#include <sys/stat.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <sys/time.h> > +#include <sys/resource.h> > +#include <sys/sysinfo.h> > +#include <sys/poll.h> > +#include <netinet/in.h> > +#include <arpa/inet.h> > +#include <netdb.h> > +#include <signal.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <pthread.h> > +#include <unistd.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <unistd.h> > +#include <string.h> > +#include <errno.h> > + > +#include "test.h" > +#include "usctest.h" > +#include "safe_macros.h" > + > +char *TCID = "tcp_fastopen"; > + > +static const int max_msg_len = 1500; > + > +/* TCP server requiers */ > +#ifndef TCP_FASTOPEN > +#define TCP_FASTOPEN 23 > +#endif > + > +/* TCP client requiers */ > +#ifndef MSG_FASTOPEN > +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ > +#endif > + > +enum { > + TCP_SERVER = 0, > + TCP_CLIENT, > +}; > +static int tcp_mode; > + > +enum { > + TFO_ENABLED = 0, > + TFO_DISABLED, > +}; > +static int tfo_support; > +static int fastopen_api; > + > +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; > +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; > +static int tw_reuse_changed; > +static int tfo_cfg_value; > +static int tfo_bit_num; > +static int tfo_cfg_changed; > +static int tfo_queue_size = 100; > +static int max_queue_len = 100; > +static int client_byte = 0x43; > +static int server_byte = 0x53; > +static const int start_byte = 0x24; > +static const int start_fin_byte = 0x25; > +static const int end_byte = 0x0a; > +static int client_msg_size = 32; > +static int server_msg_size = 128; > +static char *client_msg; > +static char *server_msg; > + > +/* > + * The number of requests from client after > + * which server has to close the connection. > + */ > +static int server_max_requests = 3; > +static int client_max_requests = 2000000; > +static int clients_num = 2; > +static char *tcp_port = "61000"; > +static char *server_addr = "localhost"; > +/* server socket */ > +static int sfd; > + > +/* how long a client must wait for the server's reply, microsec */ > +static long wait_timeout = 10000000; > + > +/* in the end test will save time result in this file */ > +static char *rpath = "./tfo_result"; > + > +static int force_run; > +static int verbose; > + > +static char *narg, *Narg, *qarg, *barg, *Barg, > + *rarg, *Rarg, *aarg, *Targ; > + > +static const option_t options[] = { > + /* server params */ > + {"R:", NULL, &Rarg}, > + {"q:", NULL, &qarg}, > + > + /* client params */ > + {"H:", NULL, &server_addr}, > + {"a:", NULL, &aarg}, > + {"n:", NULL, &narg}, > + {"b:", NULL, &barg}, > + {"N:", NULL, &Narg}, > + {"B:", NULL, &Barg}, > + {"T:", NULL, &Targ}, > + {"r:", NULL, &rarg}, > + {"d:", NULL, &rpath}, > + > + /* common */ > + {"g:", NULL, &tcp_port}, > + {"F", &force_run, NULL}, > + {"l", &tcp_mode, NULL}, > + {"o", &fastopen_api, NULL}, > + {"O", &tfo_support, NULL}, > + {"v", &verbose, NULL}, > + {NULL, NULL, NULL} > +}; > + > +static void help(void) > +{ > + printf("\n -F Force to run\n"); > + printf(" -v Verbose\n"); > + printf(" -o Use old TCP API, default is new TCP API\n"); > + printf(" -O TFO support is off, default is on\n"); > + printf(" -l Become TCP Client, default is TCP server\n"); > + printf(" -g x x - server port, default is %s\n", tcp_port); > + > + printf("\n Client:\n"); > + printf(" -H x x - server name or ip address, default is '%s'\n", > + server_addr); > + printf(" -a x x - num of clients running in parallel\n"); > + printf(" -r x x - num of client requests\n"); > + printf(" -n x Client message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -b x x is a byte of client message\n"); > + printf(" -N x Server message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -B x x is a byte of server message\n"); > + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", > + wait_timeout); > + printf(" -d x x is a path to the file where results are saved\n"); > + > + printf("\n Server:\n"); > + printf(" -R x x - num of requests, after which conn. closed\n"); > + printf(" -q x x - server's limit on the queue of TFO requests\n"); > +} > + > +/* common structure for TCP server and TCP client */ > +struct tcp_func { > + void (*init)(void); > + void (*run)(void); > + void (*cleanup)(void); > +}; > +static struct tcp_func tcp; > + > +#define MAX_THREADS 10000 > +static pthread_attr_t attr; > +static pthread_t *thread_ids; > +static int threads_num; > + > +static struct addrinfo *remote_addrinfo; > +static struct addrinfo *local_addrinfo; > +static const struct linger clo = { 1, 3 }; > + > +static void cleanup(void) > +{ > + static int first = 1; > + if (!first) > + return; > + first = 0; > + > + tst_resm(TINFO, "cleanup"); > + > + free(client_msg); > + free(server_msg); > + > + tcp.cleanup(); > + > + if (tfo_cfg_changed) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tfo_cfg_value &= ~tfo_bit_num; > + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); > + tst_resm(TINFO, "unset '%s' back to '%d'", > + tfo_cfg, tfo_cfg_value); > + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); > + } > + > + if (tw_reuse_changed) { > + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); > + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); > + } > + TEST_CLEANUP; > +} > + > +int sock_recv_poll(int *fd, char *buf, const int *buf_size, int *offset) > +{ > + struct pollfd pfd; > + pfd.fd = *fd; > + pfd.events = POLLIN; > + int len = -1; > + while (1) { > + errno = 0; > + int ret = poll(&pfd, 1, wait_timeout / 1000); > + if (ret == -1) { > + if (errno == EINTR) > + continue; > + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", > + __FILE__, __LINE__); > + break; > + } > + if (ret == 0) { > + tst_resm(TFAIL, "msg timeout, sock '%d'", *fd); > + break; > + } > + > + if (ret != 1 || !(pfd.revents & POLLIN)) > + break; > + > + errno = 0; > + len = recv(*fd, buf + *offset, > + *buf_size - *offset, MSG_DONTWAIT); > + > + if (len == -1 && errno == EINTR) > + continue; > + else > + break; > + } > + > + if (len == 0) > + tst_resm(TINFO, "sock was closed '%d'", *fd); > + > + return len; > +} > + > +void client_recv(int *fd, char *buf) > +{ > + int len, offset = 0; > + > + while (1) { > + > + len = sock_recv_poll(fd, buf, &server_msg_size, &offset); > + > + /* socket closed or msg is not valid */ > + if (len < 1 || (offset + len) > server_msg_size || > + (buf[0] != start_byte && buf[0] != start_fin_byte)) { ^ I would align this with the start of the brace at first line so that you can see at first sight that the condition continues to the next line. > + tst_resm(TFAIL | TERRNO, > + "recv failed, sock '%d'", *fd); > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) > + continue; > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *fd); > + } > + > + if (buf[0] == start_fin_byte) { > + /* recv last msg, close socket */ > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + } > + break; > + } > +} > + > +int client_connect_send(int *cfd) > +{ > + *cfd = socket(AF_INET, SOCK_STREAM, 0); > + const int flag = 1; > + setsockopt(*cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (*cfd == -1) { > + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", > + __FILE__, __LINE__); > + return -1; > + } > + > + if (fastopen_api == TFO_ENABLED) { > + /* Replaces connect() + send()/write() */ > + if (sendto(*cfd, client_msg, client_msg_size, > + MSG_FASTOPEN | MSG_NOSIGNAL, > + remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "sendto failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } else { > + /* old TCP API */ > + if (connect(*cfd, remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen)) { > + tst_resm(TFAIL | TERRNO, "connect failed"); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + > + if (send(*cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, > + "send failed on sock '%d'", *cfd); > + SAFE_CLOSE(NULL, *cfd); > + return -1; > + } > + } > + > + return 0; > +} > + > +void *client_fn(void *arg) > +{ > + char buf[server_msg_size]; > + int cfd, i; > + > + /* connect & send requests */ > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + for (i = 1; i < client_max_requests; ++i) { > + > + /* check connection, it can be closed */ > + int ret = 0; > + if (cfd != -1) > + ret = recv(cfd, buf, 1, MSG_DONTWAIT); > + > + if (ret == 0) { > + /* try to reconnect and send */ > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + > + if (client_connect_send(&cfd)) > + return NULL; > + client_recv(&cfd, buf); > + > + continue; > + > + } else if (ret > 0) { > + tst_resm_hexd(TFAIL, buf, 1, > + "received after recv, sock '%d':", cfd); > + } > + > + if (verbose) { > + tst_resm_hexd(TINFO, client_msg, client_msg_size, > + "try to send msg[%d]", i); > + } > + > + if (send(cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "send failed"); > + break; > + } > + client_recv(&cfd, buf); > + } > + > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + return NULL; > +} > + > +union uint2bytes { > + char byte[2]; > + uint16_t val; > +}; > + > +static struct timeval tv_client_start; > +static struct timeval tv_client_end; > + > +static void make_client_request(void) > +{ > + client_msg[0] = start_byte; > + > + /* reply will be filled with this byte */ > + client_msg[1] = server_byte; > + > + /* set size for reply */ > + union uint2bytes reply_size; > + reply_size.val = htons(server_msg_size); > + > + /* write requested reply size to bytes 2, 3 */ > + client_msg[2] = reply_size.byte[0]; > + client_msg[3] = reply_size.byte[1]; > + > + client_msg[client_msg_size - 1] = end_byte; > +} > + > +void parse_client_request(const char *recv_msg, uint16_t *send_msg_size, > + char *send_msg_byte) > +{ > + *send_msg_byte = recv_msg[1]; > + > + /* read requested reply size */ > + union uint2bytes reply_size; > + reply_size.byte[0] = recv_msg[2]; > + reply_size.byte[1] = recv_msg[3]; > + *send_msg_size = ntohs(reply_size.val); > + if (*send_msg_size < 2 || *send_msg_size > max_msg_len) { > + tst_resm(TWARN, "wrong msg size received %u", *send_msg_size); > + *send_msg_size = 2; > + } > +} > + > +static void client_init(void) > +{ > + if (clients_num >= MAX_THREADS) { > + tst_brkm(TBROK, cleanup, > + "Unexpected num of clients '%d'", > + clients_num); > + } > + > + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); > + > + client_msg = SAFE_MALLOC(NULL, client_msg_size); > + memset(client_msg, client_byte, client_msg_size); > + > + make_client_request(); > + > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + gettimeofday(&tv_client_start, NULL); > + int i; > + for (i = 0; i < clients_num; ++i) { > + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", > + __FILE__, __LINE__); > + } > + ++threads_num; > + } > + > +} > + > +static void client_run(void) > +{ > + void *res = NULL; > + long clnt_time = 0; > + > + int i; > + for (i = 0; i < clients_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + threads_num = 0; > + > + gettimeofday(&tv_client_end, NULL); The gettimeofday time measurement may break if someone happen to change the time while the test was running (i.e. ntp client daemon). clock_gettime(CLOCK_MONOTONIC, &tp) should be better. > + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + > + tv_client_end.tv_usec - tv_client_start.tv_usec; > + > + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); > + > + /* ask server to terminate */ > + int cfd; > + client_msg[0] = start_fin_byte; > + if (!client_connect_send(&cfd)) { > + shutdown(cfd, SHUT_WR); > + SAFE_CLOSE(NULL, cfd); > + } I would be happier if you have passed the client_connect_send() arguments as arguments instead of using global variables (it would be easier to follow the codepath). Moreover I would make the function to return the filedescriptor directly, it returns int anyway... > + /* the script tcp_fastopen_run.sh will remove it */ > + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); > +} > + > +static void client_cleanup(void) > +{ > + void *res = NULL; > + int i; > + for (i = 0; i < threads_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + free(thread_ids); > + > + if (remote_addrinfo) > + freeaddrinfo(remote_addrinfo); > +} > + > + > +void make_server_reply(char **send_msg, const uint16_t *send_msg_size, > + const char *send_msg_byte) > +{ > + *send_msg = SAFE_MALLOC(NULL, *send_msg_size); > + memset(*send_msg, *send_msg_byte, *send_msg_size - 1); > + > + (*send_msg)[0] = start_byte; > + > + (*send_msg)[*send_msg_size - 1] = end_byte; What about simply returning pointer to the newly allocated buffer instead of the ugly pointer to an array? > +} > + > +void *server_fn(void *cfd) > +{ > + int client_fd = (intptr_t) cfd; > + int num_requests = 0, offset = 0; > + > + /* Reply will be constructed from first client request */ > + char *send_msg = NULL, send_msg_byte; > + uint16_t send_msg_size = 0; > + > + char recv_msg[max_msg_len]; > + > + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); > + ssize_t recv_len; > + > + while (1) { > + errno = 0; > + recv_len = sock_recv_poll(&client_fd, recv_msg, > + &max_msg_len, &offset); > + > + if (recv_len == 0) { > + break; > + } else if (recv_len < 0 || (offset + recv_len) > max_msg_len || > + (recv_msg[0] != start_byte && > + recv_msg[0] != start_fin_byte)) { ^ I would add two more spaces here so that it's aligned with the if conditions and you can see at first sight that the condition continues to the next line. > + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); > + break; > + } > + > + offset += recv_len; > + > + if (recv_msg[offset - 1] != end_byte) { > + tst_resm(TINFO, "msg is not complete, continue recv"); > + continue; > + } > + > + if (recv_msg[0] == start_fin_byte) > + tst_brkm(TBROK, cleanup, "client asks to terminate..."); Why is this TBROK? Also looking int the tst_res() the T_exitval |= ttype_result; is not guarded by locks, so in very unlikely case the value may be rewritten by a tst_resm() for example. (Two threads enters tst_res and each of them gets the value, modifies it and saves it and the result depends on the order of these operations) And the problem is if we add a locks there, all tests would need to be compiled with -lpthread, which is someting I do not want to do. :( One solution would be not using the tst_interface in this program at all as it's executed by the shell script that prints the test messages and returns exit value... And the same would simplify the propagation of the result that is currently hacked around via the tfo_result file. You could have simply printed the value into the stdout. But on the other hand this will be more work on the testcase. > + if (verbose) { > + tst_resm_hexd(TINFO, recv_msg, offset, > + "msg recv from sock %d:", client_fd); > + } > + > + /* if we send reply for the first time, construct it here */ > + if (!send_msg) { > + parse_client_request(recv_msg, &send_msg_size, > + &send_msg_byte); > + > + make_server_reply(&send_msg, &send_msg_size, > + &send_msg_byte); > + } > + > + /* > + * It will tell client that server is going > + * to close this connection. > + */ > + if (++num_requests >= server_max_requests) > + send_msg[0] = start_fin_byte; > + > + if (send(client_fd, send_msg, send_msg_size, > + MSG_NOSIGNAL) == -1) > + tst_resm(TWARN | TERRNO, "Error while sending msg"); > + else { > + if (verbose) { > + tst_resm_hexd(TINFO, send_msg, send_msg_size, > + "msg sent:"); > + } > + } > + > + offset = 0; > + > + if (num_requests >= server_max_requests) { > + if (verbose) > + tst_resm(TINFO, "Max reqs, close socket"); > + shutdown(client_fd, SHUT_WR); > + break; > + } > + } > + > + free(send_msg); > + SAFE_CLOSE(NULL, client_fd); > + > + return NULL; > +} > + > +static void server_thread_add(intptr_t client_fd) > +{ > + pthread_t id; > + if (pthread_create(&id, &attr, server_fn, (intptr_t *) client_fd)) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", __FILE__, __LINE__); > + } > +} > + > +static void server_init(void) > +{ > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + hints.ai_flags = AI_PASSIVE; > + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + sfd = socket(AF_INET, SOCK_STREAM, 0); > + if (sfd == -1) > + tst_brkm(TBROK, cleanup, "Failed to create a socket"); > + > + tst_resm(TINFO, "assigning a name to the server socket..."); > + if (!local_addrinfo) > + tst_brkm(TBROK, cleanup, "failed to get the address"); > + > + while (bind(sfd, local_addrinfo->ai_addr, > + local_addrinfo->ai_addrlen) == -1) { > + sleep(1); I do not really like sleep(1) in testcases in most of the cases it slows down testruns and it adds up to minutes and hours for thousands of testcases... What about trying ten times in a second with usleep(100000); or similar? > + } > + tst_resm(TINFO, "the name assigned"); > + > + const int flag = 1; > + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (fastopen_api == TFO_ENABLED) { > + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, > + sizeof(tfo_queue_size)) == -1) > + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); > + } > + > + listen(sfd, max_queue_len); > + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); You can free the local_addrinfo here. > +} > + > +static void server_cleanup(void) > +{ > + SAFE_CLOSE(NULL, sfd); > + if (local_addrinfo) > + freeaddrinfo(local_addrinfo); > +} > + > +static void server_run(void) > +{ > + struct sockaddr_in client_addr; > + socklen_t addr_size = sizeof(client_addr); > + pthread_attr_init(&attr); > + > + /* > + * detaching threads allow to reclaim thread's resources > + * once a thread finishes its work. > + */ > + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); > + > + while (1) { > + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, > + &addr_size); > + if (client_fd == -1) { > + tst_brkm(TBROK, cleanup, "Can't create client socket"); > + continue; > + } > + > + if (client_addr.sin_family == AF_INET) { > + if (verbose) { > + tst_resm(TINFO, "conn: port '%d', addr '%s'", > + client_addr.sin_port, > + inet_ntoa(client_addr.sin_addr)); > + } > + } > + server_thread_add(client_fd); > + } > +} > + > +static void check_opt(const char *name, char *arg, int *val, int lim) > +{ > + if (arg) { > + if (sscanf(arg, "%i", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", > + name, lim); > + } > +} > + > +static void check_opt_l(const char *name, char *arg, long *val, long lim) > +{ > + if (arg) { > + if (sscanf(arg, "%ld", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", > + name, lim); > + } > +} > + > +static void check_msg_byte(int *byte) > +{ > + /* these bytes are reserved */ > + if (*byte == start_byte || *byte == start_fin_byte || > + *byte == end_byte) > + tst_brkm(TBROK, NULL, > + "-B option, 0x%02x is reserved", *byte); > +} > + > +/* cleanup flags */ > +void setup(int argc, char *argv[]) > +{ > + char *msg; > + msg = parse_opts(argc, argv, options, help); > + if (msg != NULL) > + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); > + > + /* if client num is not set, use num of processors */ > + struct sysinfo si; > + if (sysinfo(&si) == 0) > + clients_num = si.procs; Call to sysconf(_SC_NPROCESSORS_ONLN) would be better choice here. > + check_opt("a", aarg, &clients_num, 1); > + check_opt("r", rarg, &client_max_requests, 1); > + check_opt("R", Rarg, &server_max_requests, 1); > + check_opt("n", narg, &client_msg_size, 1); > + > + check_opt("b", barg, &client_byte, 0); > + check_msg_byte(&client_byte); > + > + check_opt("N", Narg, &server_msg_size, 1); > + > + check_opt("B", Barg, &server_byte, 0); > + check_msg_byte(&server_byte); Does chaning the message content bytes actually adds up some value or not? Do you expect that test outcome would be different with different values? If not, I would remove this options, the test is complex enough itself. > + check_opt("q", qarg, &tfo_queue_size, 1); > + check_opt_l("T", Targ, &wait_timeout, 0L); > + > + if (!force_run) > + tst_require_root(NULL); > + > + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { > + tst_brkm(TCONF, NULL, > + "Test must be run with kernel 3.7 or newer"); > + } > + > + /* check tcp fast open knob */ > + if (!force_run && access(tfo_cfg, F_OK) == -1) > + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); > + > + if (!force_run) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); > + } > + > + tst_sig(FORK, DEF_HANDLER, cleanup); > + > + tst_resm(TINFO, "TCP %s is using %s TCP API.", > + (tcp_mode == TCP_SERVER) ? "server" : "client", > + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); > + > + switch (tcp_mode) { > + case TCP_SERVER: > + tst_resm(TINFO, "max requests '%d'", > + server_max_requests); > + tcp.init = server_init; > + tcp.run = server_run; > + tcp.cleanup = server_cleanup; > + tfo_bit_num = 2; > + break; > + case TCP_CLIENT: > + tst_resm(TINFO, "connection: %s:%s", > + server_addr, tcp_port); > + tst_resm(TINFO, "client max req: %d", client_max_requests); > + tst_resm(TINFO, "clients num: %d", clients_num); > + tst_resm(TINFO, "client msg size: %d", client_msg_size); > + tst_resm(TINFO, "server msg size: %d", server_msg_size); > + tst_resm(TINFO, "client msg byte: %02x", client_byte); > + tst_resm(TINFO, "server msg byte: %02x", server_byte); > + > + tcp.init = client_init; > + tcp.run = client_run; > + tcp.cleanup = client_cleanup; > + tfo_bit_num = 1; > + break; > + } > + > + tfo_support = TFO_ENABLED == tfo_support; > + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { > + int value = (tfo_cfg_value & ~tfo_bit_num) > + | (tfo_support << (tfo_bit_num - 1)); > + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); > + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); > + tfo_cfg_changed = 1; > + } > + > + int reuse_value = 0; > + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); > + if (!reuse_value) { > + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); > + tw_reuse_changed = 1; > + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); > + } > + > + tst_resm(TINFO, "TFO support %s", > + (tfo_support) ? "enabled" : "disabled"); > + > + tcp.init(); > +} > + > +int main(int argc, char *argv[]) > +{ > + setup(argc, argv); > + > + tcp.run(); > + > + cleanup(); > + > + tst_exit(); > +} > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > new file mode 100755 > index 0000000..ae77b58 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > @@ -0,0 +1,173 @@ > +#!/bin/bash > + > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > +# > +# Author: Alexey Kodanev <ale...@or...> > +# > + > +# default command-line options > +user_name="root" > +remote_addr=$RHOST > +use_ssh=0 > +clients_num=2 > +client_requests=2000000 > +max_requests=3 > +server_port=$[ $RANDOM % 28232 + 32768 ] > + > +while getopts :hu:H:sr:p:n:R: opt; do > + case "$opt" in > + h) > + echo "Usage:" > + echo "h help" > + echo "u x server user name" > + echo "H x server hostname or IP address" > + echo "s use ssh to run remote cmds" > + echo "n x num of clients running in parallel" > + echo "r x the number of client requests" > + echo "R x num of requests, after which conn. closed" > + echo "p x server port" > + exit 0 > + ;; > + u) user_name=$OPTARG ;; > + H) remote_addr=$OPTARG ;; > + s) use_ssh=1 ;; > + n) clients_num=$OPTARG ;; > + r) client_requests=$OPTARG ;; > + R) max_requests=$OPTARG ;; > + p) server_port=$OPTARG ;; > + *) > + tst_brkm TBROK NULL "unknown option: $opt" > + exit 2 > + ;; > + esac > +done > + > +run_remote_cmd() > +{ > + tst_resm TINFO "run cmd on $remote_addr: $1" > + > + if [ "$use_ssh" = 1 ]; then > + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" > + else > + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" > + fi > +} > + > +cleanup() > +{ > + rm -f $tfo_result > + run_remote_cmd "pkill -9 tcp_fastopen\$" > +} > + > +read_result_file() > +{ > + if [ -f $tfo_result ]; then > + if [ -r $tfo_result ]; then > + cat $tfo_result > + else > + tst_brkm TBROK NULL "Failed to read result file" > + exit 2 > + fi > + else > + tst_brkm TBROK NULL "Failed to find result file" > + exit 2 > + fi > +} > + > +check_exit_status() > +{ > + if [ "$1" -ne "0" ]; then > + tst_brkm TBROK NULL "Last test has failed" > + exit $1; > + fi > +} > + > +export RC=0 > +export TST_TOTAL=1 > +export TCID="tcp_fastopen" > +export TST_COUNT=0 > + > +tfo_result_ms=0 > +bind_timeout=30 > + > + > +tst_kvercmp 3 7 0 > +if [ $? -eq 0 ]; then > + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" > + exit 0 > +fi > + > +if [ -z $remote_addr ]; then > + tst_brkm TBROK NULL "you must specify server address" > + exit 2 > +fi > + > +tdir="${LTPROOT}/testcases/bin/" > +tfo_result="${TMPDIR}/tfo_result" > + > +trap "cleanup" EXIT > + > +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" > + > +if [ "$use_ssh" = 1 ]; then > + scp -q ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > +else > + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir > +fi Eh, you copy the test binary to the remote machine and then execute it? The rest of the network testcases expects that ltp is installed both on the client and the server machine and while I don't like that it expects that the LTP install directory is /opt/ltp it's far more robust than this. For example x86_64 machine is on one end and i386 on the other one will not work this way. > +# Run test without/with TFO > +tcp_api_opt=("-o -O" "") > + > +# Time results for without/with TFO test > +vtime=(0 0) > + > +for (( i = 0; i < 2; ++i )); do > + # kill tcp server on remote machine > + run_remote_cmd "pkill -9 tcp_fastopen\$" > + > + sleep 2 > + > + # run tcp server on remote machine > + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ > +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" > + > + sleep $bind_timeout > + > + # run local tcp client > + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ > +-H $remote_addr ${tcp_api_opt[$i]} -g $server_port -d $tfo_result > + > + check_exit_status $? > + > + vtime[$i]=`read_result_file` > + > + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then > + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" > + exit 2 > + fi > + server_port=$[ $server_port + 1 ] > +done > + > +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] > + > +if (( $tfo_cmp < 3 )); then > + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" > + exit 1 > +fi > + > +tst_resm TPASS "TFO performance result is $tfo_cmp percent" > +exit 0 -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2013-11-26 10:52:15
|
Hi! On 25.11.2013 22:40, ch...@su... wrote: > Hi! >> + >> +} >> + >> +static void client_run(void) >> +{ >> + void *res = NULL; >> + long clnt_time = 0; >> + >> + int i; >> + for (i = 0; i < clients_num; ++i) >> + pthread_join(thread_ids[i], &res); >> + >> + threads_num = 0; >> + >> + gettimeofday(&tv_client_end, NULL); > The gettimeofday time measurement may break if someone happen to change > the time while the test was running (i.e. ntp client daemon). > > clock_gettime(CLOCK_MONOTONIC, &tp) > > should be better. The clock_gettime manual says that CLOCK_MONOTONIC_RAW similar to _MONOTONIC, but is not subject to NTP adjustments. So CLOCK_MONOTONIC can be adjusted, doesn't it? > >> + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000000 + >> + tv_client_end.tv_usec - tv_client_start.tv_usec; >> + >> + tst_resm(TINFO, "total time '%ld' ms", clnt_time / 1000); >> + >> + /* ask server to terminate */ >> + int cfd; >> + client_msg[0] = start_fin_byte; >> + if (!client_connect_send(&cfd)) { >> + shutdown(cfd, SHUT_WR); >> + SAFE_CLOSE(NULL, cfd); >> + } > I would be happier if you have passed the client_connect_send() > arguments as arguments instead of using global variables (it would be > easier to follow the codepath). Moreover I would make the function to > return the filedescriptor directly, it returns int anyway... OK > >> + /* the script tcp_fastopen_run.sh will remove it */ >> + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time / 1000); >> +} >> + >> +static void client_cleanup(void) >> +{ >> + void *res = NULL; >> + int i; >> + for (i = 0; i < threads_num; ++i) >> + pthread_join(thread_ids[i], &res); >> + >> + free(thread_ids); >> + >> + if (remote_addrinfo) >> + freeaddrinfo(remote_addrinfo); >> +} >> + >> + >> +void make_server_reply(char **send_msg, const uint16_t *send_msg_size, >> + const char *send_msg_byte) >> +{ >> + *send_msg = SAFE_MALLOC(NULL, *send_msg_size); >> + memset(*send_msg, *send_msg_byte, *send_msg_size - 1); >> + >> + (*send_msg)[0] = start_byte; >> + >> + (*send_msg)[*send_msg_size - 1] = end_byte; > What about simply returning pointer to the newly allocated buffer > instead of the ugly pointer to an array? Oh, yes thanks. It will be much nicer and simpler. >> + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); >> + break; >> + } >> + >> + offset += recv_len; >> + >> + if (recv_msg[offset - 1] != end_byte) { >> + tst_resm(TINFO, "msg is not complete, continue recv"); >> + continue; >> + } >> + >> + if (recv_msg[0] == start_fin_byte) >> + tst_brkm(TBROK, cleanup, "client asks to terminate..."); > Why is this TBROK? Thought about all in one function, which prints description messages, calls cleanup function and exit the program. > > Also looking int the tst_res() the T_exitval |= ttype_result; is not > guarded by locks, so in very unlikely case the value may be rewritten by > a tst_resm() for example. (Two threads enters tst_res and each of them > gets the value, modifies it and saves it and the result depends on the > order of these operations) > > And the problem is if we add a locks there, all tests would need to be > compiled with -lpthread, which is someting I do not want to do. :( > > One solution would be not using the tst_interface in this program at all > as it's executed by the shell script that prints the test messages and > returns exit value... And the same would simplify the propagation of the > result that is currently hacked around via the tfo_result file. You > could have simply printed the value into the stdout. But on the other > hand this will be more work on the testcase. What about built in gcc functions for atomic memory access like |__sync_fetch_and_or()?| >> + >> + >> + while (bind(sfd, local_addrinfo->ai_addr, >> + local_addrinfo->ai_addrlen) == -1) { >> + sleep(1); > I do not really like sleep(1) in testcases in most of the cases it slows > down testruns and it adds up to minutes and hours for thousands of > testcases... What about trying ten times in a second with > usleep(100000); or similar? OK > ... > >> + check_opt("a", aarg, &clients_num, 1); >> + check_opt("r", rarg, &client_max_requests, 1); >> + check_opt("R", Rarg, &server_max_requests, 1); >> + check_opt("n", narg, &client_msg_size, 1); >> + >> + check_opt("b", barg, &client_byte, 0); >> + check_msg_byte(&client_byte); >> + >> + check_opt("N", Narg, &server_msg_size, 1); >> + >> + check_opt("B", Barg, &server_byte, 0); >> + check_msg_byte(&server_byte); > Does chaning the message content bytes actually adds up some value or > not? Do you expect that test outcome would be different with different > values? If not, I would remove this options, the test is complex enough > itself. True, I will set some constant. > >> + >> +run_remote_cmd "[ ! -e $tdir ] && mkdir -p $tdir" >> + >> +if [ "$use_ssh" = 1 ]; then >> + scp -q ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir >> +else >> + rcp ${tdir}tcp_fastopen $user_name@$remote_addr:$tdir >> +fi > Eh, you copy the test binary to the remote machine and then execute it? > > The rest of the network testcases expects that ltp is installed both on > the client and the server machine and while I don't like that it expects > that the LTP install directory is /opt/ltp it's far more robust than > this. For example x86_64 machine is on one end and i386 on the other one > will not work this way. OK, I see it now, will remove it. |
From: <ch...@su...> - 2013-11-26 14:16:58
|
Hi! > > The gettimeofday time measurement may break if someone happen to change > > the time while the test was running (i.e. ntp client daemon). > > > > clock_gettime(CLOCK_MONOTONIC, &tp) > > > > should be better. > The clock_gettime manual says that CLOCK_MONOTONIC_RAW similar to > _MONOTONIC, but is not subject to NTP adjustments. So CLOCK_MONOTONIC > can be adjusted, doesn't it? Right, the CLOCK_MONOTONIC_RAW is better choice. > >> + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); > >> + break; > >> + } > >> + > >> + offset += recv_len; > >> + > >> + if (recv_msg[offset - 1] != end_byte) { > >> + tst_resm(TINFO, "msg is not complete, continue recv"); > >> + continue; > >> + } > >> + > >> + if (recv_msg[0] == start_fin_byte) > >> + tst_brkm(TBROK, cleanup, "client asks to terminate..."); > > Why is this TBROK? > Thought about all in one function, which prints description messages, > calls cleanup function and exit the program. I would say this is a misuse of the API as it is... I suppose that you can do this in the server because the return value is not checked, but it's confusing when reading the code. > > guarded by locks, so in very unlikely case the value may be rewritten by > > a tst_resm() for example. (Two threads enters tst_res and each of them > > gets the value, modifies it and saves it and the result depends on the > > order of these operations) > > > > And the problem is if we add a locks there, all tests would need to be > > compiled with -lpthread, which is someting I do not want to do. :( > > > > One solution would be not using the tst_interface in this program at all > > as it's executed by the shell script that prints the test messages and > > returns exit value... And the same would simplify the propagation of the > > result that is currently hacked around via the tfo_result file. You > > could have simply printed the value into the stdout. But on the other > > hand this will be more work on the testcase. > What about built in gcc functions for atomic memory access like > |__sync_fetch_and_or()?| That would be easy solution, but I doubt that these are available on all platforms and slightly older compilers. Perpahs we can add configure checks and enable it only when it's supported. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2013-11-27 12:06:51
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/README | 16 + testcases/network/tcp_fastopen/tcp_fastopen.c | 783 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 164 ++++ 5 files changed, 988 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/README create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..769364f --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread -lrt + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/README b/testcases/network/tcp_fastopen/README new file mode 100644 index 0000000..4e19add --- /dev/null +++ b/testcases/network/tcp_fastopen/README @@ -0,0 +1,16 @@ +TCP Fast Open (TFO) is an extension to speed up the opening of TCP connections +between two endpoints. It reduces the number of round time trips (RTT) +required in TCP conversations. + +The test must be run on Linux versions higher then 3.7. (client side +implemented in Linux 3.6, server side in Linux 3.7). + +Developers reported (in "TCP Fast Open", CoNEXT 2011), TFO could result in +speed improvements of between 4% and 41% in the page load times on popular web +sites. + +With the default test scenario, a client is constantly sending requests to +a server and waiting for replies. The server is closing a connection after 3 +requests from the client. This scenario was chosen to simulate an average +conversation between a web-browser and an application server. So the test +results with TFO support enabled must be at least 3 percent faster. diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..05a38cd --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,783 @@ +/* + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/sysinfo.h> +#include <sys/poll.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static const int client_byte = 0x43; +static const int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"N:", NULL, &Narg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; +static int threads_num; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void cleanup(void) +{ + static int first = 1; + if (!first) + return; + first = 0; + + tst_resm(TINFO, "cleanup"); + + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} + +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", + __FILE__, __LINE__); + break; + } + if (ret == 0) { + tst_resm(TFAIL, "msg timeout, sock '%d'", fd); + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(fd, buf + *offset, + buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + if (len == 0) + tst_resm(TINFO, "sock was closed '%d'", fd); + + return len; +} + +static void client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); + + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + tst_resm(TFAIL | TERRNO, + "recv failed, sock '%d'", *fd); + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + break; + } + + offset += len; + + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + if (buf[0] == start_fin_byte) { + /* recv last msg, close socket */ + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(NULL, *fd); + *fd = -1; + } + break; + } +} + +static int client_connect_send(const char *msg, int size) +{ + int cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (cfd == -1) { + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", + __FILE__, __LINE__); + return cfd; + } + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != size) { + tst_resm(TFAIL | TERRNO, "sendto failed"); + SAFE_CLOSE(NULL, cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + tst_resm(TFAIL | TERRNO, "connect failed"); + SAFE_CLOSE(NULL, cfd); + return -1; + } + + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, + "send failed on sock '%d'", cfd); + SAFE_CLOSE(NULL, cfd); + return -1; + } + } + + return cfd; +} + +void *client_fn(void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + + /* connect & send requests */ + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) + return NULL; + client_recv(&cfd, buf); + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) + return NULL; + client_recv(&cfd, buf); + + continue; + + } else if (ret > 0) { + tst_resm_hexd(TFAIL, buf, 1, + "received after recv, sock '%d':", cfd); + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + tst_resm(TFAIL | TERRNO, "send failed"); + break; + } + client_recv(&cfd, buf); + } + + if (cfd != -1) + SAFE_CLOSE(NULL, cfd); + return NULL; +} + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* set size for reply */ + *(uint16_t *)(client_msg + 1) = htons(server_msg_size); + client_msg[client_msg_size - 1] = end_byte; +} + +static int parse_client_request(const char *recv_msg) +{ + int size = ntohs(*(uint16_t *)(recv_msg + 1)); + + if (size < 2 || size > max_msg_len) { + tst_resm(TWARN, "wrong msg size received %u", size); + size = 2; + } + + return size; +} + +static struct timespec tv_client_start; +static struct timespec tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + ++threads_num; + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + + int i; + for (i = 0; i < clients_num; ++i) + pthread_join(thread_ids[i], &res); + + threads_num = 0; + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time); + + /* ask server to terminate */ + client_msg[0] = start_fin_byte; + int cfd = client_connect_send(client_msg, client_msg_size); + if (cfd != -1) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); +} + +static void client_cleanup(void) +{ + void *res = NULL; + int i; + for (i = 0; i < threads_num; ++i) + pthread_join(thread_ids[i], &res); + + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + + +static char *make_server_reply(int size) +{ + char *send_msg = SAFE_MALLOC(NULL, size); + memset(send_msg, server_byte, size - 1); + send_msg[0] = start_byte; + send_msg[size - 1] = end_byte; + + return send_msg; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL; + int send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + recv_len = sock_recv_poll(client_fd, recv_msg, + max_msg_len, &offset); + + if (recv_len == 0) { + break; + } else + if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); + break; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + tst_resm(TINFO, "msg is not complete, continue recv"); + continue; + } + + if (recv_msg[0] == start_fin_byte) + tst_brkm(TBROK, cleanup, "client asks to terminate..."); + + if (verbose) { + tst_resm_hexd(TINFO, recv_msg, offset, + "msg recv from sock %d:", client_fd); + } + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + send_msg_size = parse_client_request(recv_msg); + send_msg = make_server_reply(send_msg_size); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) { + tst_resm(TWARN | TERRNO, "Error while sending msg"); + } else if (verbose) { + tst_resm_hexd(TINFO, send_msg, send_msg_size, + "msg sent:"); + } + + offset = 0; + + if (num_requests >= server_max_requests) { + if (verbose) + tst_resm(TINFO, "Max reqs, close socket"); + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + SAFE_CLOSE(NULL, client_fd); + + return NULL; +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + usleep(100000); + } + tst_resm(TINFO, "the name assigned"); + + freeaddrinfo(local_addrinfo); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) { + tst_brkm(TBROK, cleanup, "Can't create client socket"); + continue; + } + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client num is not set, use num of processors */ + clients_num = sysconf(_SC_NPROCESSORS_ONLN); + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + check_opt("N", Narg, &server_msg_size, 1); + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..f329a0e --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 +server_port=$[ $RANDOM % 28232 + 32768 ] + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + echo "p x server port" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + p) server_port=$OPTARG ;; + *) + tst_brkm TBROK NULL "unknown option: $opt" + exit 2 + ;; + esac +done + +run_remote_cmd() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + + if [ "$use_ssh" = 1 ]; then + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" + else + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" + fi +} + +cleanup() +{ + rm -f $tfo_result + run_remote_cmd "pkill -9 tcp_fastopen\$" +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK NULL "Failed to read result file" + exit 2 + fi + else + tst_brkm TBROK NULL "Failed to find result file" + exit 2 + fi +} + +check_exit_status() +{ + if [ "$1" -ne "0" ]; then + tst_brkm TBROK NULL "Last test has failed" + exit $1; + fi +} + +export RC=0 +export TST_TOTAL=1 +export TCID="tcp_fastopen" +export TST_COUNT=0 + +tfo_result_ms=0 +bind_timeout=30 + +tst_kvercmp 3 7 0 +if [ $? -eq 0 ]; then + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" + exit 0 +fi + +if [ -z $remote_addr ]; then + tst_brkm TBROK NULL "you must specify server address" + exit 2 +fi + +tdir="${LTPROOT}/testcases/bin/" +tfo_result="${TMPDIR}/tfo_result" + +trap "cleanup" EXIT + +# Run test without/with TFO +tcp_api_opt=("-o -O" "") + +# Time results for without/with TFO test +vtime=(0 0) + +for (( i = 0; i < 2; ++i )); do + # kill tcp server on remote machine + run_remote_cmd "pkill -9 tcp_fastopen\$" + + sleep 2 + + # run tcp server on remote machine + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" + + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr ${tcp_api_opt[$i]} -g $server_port -d $tfo_result + + check_exit_status $? + + vtime[$i]=`read_result_file` + + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" + exit 2 + fi + server_port=$[ $server_port + 1 ] +done + +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] + +if (( $tfo_cmp < 3 )); then + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" + exit 1 +fi + +tst_resm TPASS "TFO performance result is $tfo_cmp percent" +exit 0 -- 1.7.1 |
From: <ch...@su...> - 2013-12-12 16:39:50
|
Hi! > This is a perfomance test for TCP Fast Open (TFO) which is an extension to > speed up the opening of TCP connections between two endpoints. It reduces > the number of round time trips (RTT) required in TCP conversations. TFO could > result in speed improvements of between 4% and 41% in the page load times on > popular web sites. > > The default test scenario simulates an average conversation between a > web-browser and an application server, so the test results with TFO enabled > must be at least 3 percent faster. > > The test must be run on Linux versions higher then 3.7. (TFO client side > implemented in Linux 3.6, server side in Linux 3.7). > > Signed-off-by: Alexey Kodanev <ale...@or...> > --- > testcases/network/tcp_fastopen/.gitignore | 1 + > testcases/network/tcp_fastopen/Makefile | 24 + > testcases/network/tcp_fastopen/README | 16 + > testcases/network/tcp_fastopen/tcp_fastopen.c | 783 ++++++++++++++++++++ > testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 164 ++++ > 5 files changed, 988 insertions(+), 0 deletions(-) > create mode 100644 testcases/network/tcp_fastopen/.gitignore > create mode 100644 testcases/network/tcp_fastopen/Makefile > create mode 100644 testcases/network/tcp_fastopen/README > create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c > create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh > > diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore > new file mode 100644 > index 0000000..f321cf0 > --- /dev/null > +++ b/testcases/network/tcp_fastopen/.gitignore > @@ -0,0 +1 @@ > +/tcp_fastopen > diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile > new file mode 100644 > index 0000000..769364f > --- /dev/null > +++ b/testcases/network/tcp_fastopen/Makefile > @@ -0,0 +1,24 @@ > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + > +top_srcdir ?= ../../.. > + > +include $(top_srcdir)/include/mk/testcases.mk > + > +INSTALL_TARGETS := tcp_fastopen_run.sh > +LDLIBS += -lpthread -lrt > + > +include $(top_srcdir)/include/mk/generic_leaf_target.mk > diff --git a/testcases/network/tcp_fastopen/README b/testcases/network/tcp_fastopen/README > new file mode 100644 > index 0000000..4e19add > --- /dev/null > +++ b/testcases/network/tcp_fastopen/README > @@ -0,0 +1,16 @@ > +TCP Fast Open (TFO) is an extension to speed up the opening of TCP connections > +between two endpoints. It reduces the number of round time trips (RTT) > +required in TCP conversations. > + > +The test must be run on Linux versions higher then 3.7. (client side > +implemented in Linux 3.6, server side in Linux 3.7). > + > +Developers reported (in "TCP Fast Open", CoNEXT 2011), TFO could result in > +speed improvements of between 4% and 41% in the page load times on popular web > +sites. > + > +With the default test scenario, a client is constantly sending requests to > +a server and waiting for replies. The server is closing a connection after 3 > +requests from the client. This scenario was chosen to simulate an average > +conversation between a web-browser and an application server. So the test > +results with TFO support enabled must be at least 3 percent faster. > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c > new file mode 100644 > index 0000000..05a38cd > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c > @@ -0,0 +1,783 @@ > +/* > + * Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > + * > + * This program is free software; you can redistribute it and/or > + * modify it under the terms of the GNU General Public License as > + * published by the Free Software Foundation; either version 2 of > + * the License, or (at your option) any later version. > + * > + * This program is distributed in the hope that it would be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, write the Free Software Foundation, > + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > + * > + * Author: Alexey Kodanev <ale...@or...> > + * > + */ > + > +#include <sys/stat.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <sys/time.h> > +#include <sys/resource.h> > +#include <sys/sysinfo.h> > +#include <sys/poll.h> > +#include <netinet/in.h> > +#include <arpa/inet.h> > +#include <netdb.h> > +#include <signal.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <pthread.h> > +#include <unistd.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <unistd.h> > +#include <string.h> > +#include <errno.h> > + > +#include "test.h" > +#include "usctest.h" > +#include "safe_macros.h" > + > +char *TCID = "tcp_fastopen"; > + > +static const int max_msg_len = 1500; > + > +/* TCP server requiers */ > +#ifndef TCP_FASTOPEN > +#define TCP_FASTOPEN 23 > +#endif > + > +/* TCP client requiers */ > +#ifndef MSG_FASTOPEN > +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ > +#endif > + > +enum { > + TCP_SERVER = 0, > + TCP_CLIENT, > +}; > +static int tcp_mode; > + > +enum { > + TFO_ENABLED = 0, > + TFO_DISABLED, > +}; > +static int tfo_support; > +static int fastopen_api; > + > +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; > +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; > +static int tw_reuse_changed; > +static int tfo_cfg_value; > +static int tfo_bit_num; > +static int tfo_cfg_changed; > +static int tfo_queue_size = 100; > +static int max_queue_len = 100; > +static const int client_byte = 0x43; > +static const int server_byte = 0x53; > +static const int start_byte = 0x24; > +static const int start_fin_byte = 0x25; > +static const int end_byte = 0x0a; > +static int client_msg_size = 32; > +static int server_msg_size = 128; > +static char *client_msg; > +static char *server_msg; > + > +/* > + * The number of requests from client after > + * which server has to close the connection. > + */ > +static int server_max_requests = 3; > +static int client_max_requests = 10; > +static int clients_num = 2; > +static char *tcp_port = "61000"; > +static char *server_addr = "localhost"; > +/* server socket */ > +static int sfd; > + > +/* how long a client must wait for the server's reply, microsec */ > +static long wait_timeout = 10000000; > + > +/* in the end test will save time result in this file */ > +static char *rpath = "./tfo_result"; > +static int force_run; > +static int verbose; > + > +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; > + > +static const option_t options[] = { > + /* server params */ > + {"R:", NULL, &Rarg}, > + {"q:", NULL, &qarg}, > + > + /* client params */ > + {"H:", NULL, &server_addr}, > + {"a:", NULL, &aarg}, > + {"n:", NULL, &narg}, > + {"N:", NULL, &Narg}, > + {"T:", NULL, &Targ}, > + {"r:", NULL, &rarg}, > + {"d:", NULL, &rpath}, > + > + /* common */ > + {"g:", NULL, &tcp_port}, > + {"F", &force_run, NULL}, > + {"l", &tcp_mode, NULL}, > + {"o", &fastopen_api, NULL}, > + {"O", &tfo_support, NULL}, > + {"v", &verbose, NULL}, > + {NULL, NULL, NULL} > +}; > + > +static void help(void) > +{ > + printf("\n -F Force to run\n"); > + printf(" -v Verbose\n"); > + printf(" -o Use old TCP API, default is new TCP API\n"); > + printf(" -O TFO support is off, default is on\n"); > + printf(" -l Become TCP Client, default is TCP server\n"); > + printf(" -g x x - server port, default is %s\n", tcp_port); > + > + printf("\n Client:\n"); > + printf(" -H x x - server name or ip address, default is '%s'\n", > + server_addr); > + printf(" -a x x - num of clients running in parallel\n"); > + printf(" -r x x - num of client requests\n"); > + printf(" -n x Client message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -N x Server message size, max msg size is '%d'\n", > + max_msg_len); > + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", > + wait_timeout); > + printf(" -d x x is a path to the file where results are saved\n"); > + > + printf("\n Server:\n"); > + printf(" -R x x - num of requests, after which conn. closed\n"); > + printf(" -q x x - server's limit on the queue of TFO requests\n"); > +} > + > +/* common structure for TCP server and TCP client */ > +struct tcp_func { > + void (*init)(void); > + void (*run)(void); > + void (*cleanup)(void); > +}; > +static struct tcp_func tcp; > + > +#define MAX_THREADS 10000 > +static pthread_attr_t attr; > +static pthread_t *thread_ids; > +static int threads_num; > + > +static struct addrinfo *remote_addrinfo; > +static struct addrinfo *local_addrinfo; > +static const struct linger clo = { 1, 3 }; > + > +static void cleanup(void) > +{ > + static int first = 1; > + if (!first) > + return; > + first = 0; There is still chance for a race condition here. But this could be easily fixed by pthread_once(). static pthread_once_t cleanup_executed = PTHREAD_ONCE_INIT; static void do_cleanup(void) { /* Do the cleanup here */ } static void cleanup(void) { pthread_once(&cleanup_executed, do_cleanup); } Thinking of the problems with LTP library thread safety again. One of the solutions may be having two tst_res.c implementaions. Note that the source of thread safety problems is the cleanup function which is executed by the tst_brkm() and the exitval that is in the test library too. If we did so then we will have tst_res.c we have now and one providing thread safe implemenation using pthread synchronization primitives (likely both will be implemented on the top of the common code). And if we could switch which one gets linked to a test at the linker phase depending whether it uses pthreads or not the thread safety problem will be fixed once for all. I will think about this a bit more. > + tst_resm(TINFO, "cleanup"); > + > + free(client_msg); > + free(server_msg); > + > + tcp.cleanup(); > + > + if (tfo_cfg_changed) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tfo_cfg_value &= ~tfo_bit_num; > + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); > + tst_resm(TINFO, "unset '%s' back to '%d'", > + tfo_cfg, tfo_cfg_value); > + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); > + } > + > + if (tw_reuse_changed) { > + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); > + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); > + } > + TEST_CLEANUP; > +} > + > +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) > +{ > + struct pollfd pfd; > + pfd.fd = fd; > + pfd.events = POLLIN; > + int len = -1; > + while (1) { > + errno = 0; > + int ret = poll(&pfd, 1, wait_timeout / 1000); > + if (ret == -1) { > + if (errno == EINTR) > + continue; > + tst_resm(TFAIL | TERRNO, "poll failed at %s:%d", > + __FILE__, __LINE__); > + break; > + } > + if (ret == 0) { > + tst_resm(TFAIL, "msg timeout, sock '%d'", fd); > + break; > + } > + > + if (ret != 1 || !(pfd.revents & POLLIN)) > + break; > + > + errno = 0; > + len = recv(fd, buf + *offset, > + buf_size - *offset, MSG_DONTWAIT); > + > + if (len == -1 && errno == EINTR) > + continue; > + else > + break; > + } > + > + if (len == 0) > + tst_resm(TINFO, "sock was closed '%d'", fd); > + > + return len; > +} > + > +static void client_recv(int *fd, char *buf) > +{ > + int len, offset = 0; > + > + while (1) { > + > + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); > + > + /* socket closed or msg is not valid */ > + if (len < 1 || (offset + len) > server_msg_size || > + (buf[0] != start_byte && buf[0] != start_fin_byte)) { > + tst_resm(TFAIL | TERRNO, > + "recv failed, sock '%d'", *fd); > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + break; > + } > + > + offset += len; > + > + if (buf[offset - 1] != end_byte) > + continue; > + > + if (verbose) { > + tst_resm_hexd(TINFO, buf, offset, > + "msg recv from sock %d:", *fd); > + } > + > + if (buf[0] == start_fin_byte) { > + /* recv last msg, close socket */ > + shutdown(*fd, SHUT_WR); > + SAFE_CLOSE(NULL, *fd); > + *fd = -1; > + } > + break; > + } > +} > + > +static int client_connect_send(const char *msg, int size) > +{ > + int cfd = socket(AF_INET, SOCK_STREAM, 0); > + const int flag = 1; > + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (cfd == -1) { > + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", > + __FILE__, __LINE__); > + return cfd; > + } > + > + if (fastopen_api == TFO_ENABLED) { > + /* Replaces connect() + send()/write() */ > + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, > + remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen) != size) { > + tst_resm(TFAIL | TERRNO, "sendto failed"); > + SAFE_CLOSE(NULL, cfd); > + return -1; > + } > + } else { > + /* old TCP API */ > + if (connect(cfd, remote_addrinfo->ai_addr, > + remote_addrinfo->ai_addrlen)) { > + tst_resm(TFAIL | TERRNO, "connect failed"); > + SAFE_CLOSE(NULL, cfd); > + return -1; > + } > + > + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, > + "send failed on sock '%d'", cfd); > + SAFE_CLOSE(NULL, cfd); > + return -1; > + } > + } > + > + return cfd; > +} > + > +void *client_fn(void *arg) > +{ > + char buf[server_msg_size]; > + int cfd, i; > + > + /* connect & send requests */ > + cfd = client_connect_send(client_msg, client_msg_size); > + if (cfd == -1) > + return NULL; So if the connection breaks after the server was started and the client fails to connect the test will run fine and the resoult would be two nearly identical runtimes? Shouldn't this rather be tst_brkm(TBROK, ...) ? > + client_recv(&cfd, buf); > + > + for (i = 1; i < client_max_requests; ++i) { > + > + /* check connection, it can be closed */ > + int ret = 0; > + if (cfd != -1) > + ret = recv(cfd, buf, 1, MSG_DONTWAIT); > + > + if (ret == 0) { > + /* try to reconnect and send */ > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + > + cfd = client_connect_send(client_msg, client_msg_size); > + if (cfd == -1) > + return NULL; > + client_recv(&cfd, buf); > + > + continue; > + > + } else if (ret > 0) { > + tst_resm_hexd(TFAIL, buf, 1, > + "received after recv, sock '%d':", cfd); > + } > + > + if (verbose) { > + tst_resm_hexd(TINFO, client_msg, client_msg_size, > + "try to send msg[%d]", i); > + } > + > + if (send(cfd, client_msg, client_msg_size, > + MSG_NOSIGNAL) != client_msg_size) { > + tst_resm(TFAIL | TERRNO, "send failed"); > + break; > + } > + client_recv(&cfd, buf); > + } > + > + if (cfd != -1) > + SAFE_CLOSE(NULL, cfd); > + return NULL; > +} > + > +static void make_client_request(void) > +{ > + client_msg[0] = start_byte; > + > + /* set size for reply */ > + *(uint16_t *)(client_msg + 1) = htons(server_msg_size); This will generate unaligned access on some architectures. What you need is to set the value byte by byte: client_msg[1] = (sever_msg_size>>8) & 0xff; client_msg[2] = sever_msg_size & 0xff; > + client_msg[client_msg_size - 1] = end_byte; > +} > + > +static int parse_client_request(const char *recv_msg) > +{ > + int size = ntohs(*(uint16_t *)(recv_msg + 1)); Here as well, you need to construct the value byte by byte as: uint16_t val = htons(((((uint16_t)recv_msg[1])<<8) + recv_msg[2])); > + if (size < 2 || size > max_msg_len) { > + tst_resm(TWARN, "wrong msg size received %u", size); > + size = 2; > + } > + > + return size; > +} > + > +static struct timespec tv_client_start; > +static struct timespec tv_client_end; > + > +static void client_init(void) > +{ > + if (clients_num >= MAX_THREADS) { > + tst_brkm(TBROK, cleanup, > + "Unexpected num of clients '%d'", > + clients_num); > + } > + > + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); > + > + client_msg = SAFE_MALLOC(NULL, client_msg_size); > + memset(client_msg, client_byte, client_msg_size); > + > + make_client_request(); > + > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); > + int i; > + for (i = 0; i < clients_num; ++i) { > + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", > + __FILE__, __LINE__); > + } > + ++threads_num; > + } > +} > + > +static void client_run(void) > +{ > + void *res = NULL; > + long clnt_time = 0; > + > + int i; > + for (i = 0; i < clients_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + threads_num = 0; > + > + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); > + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + > + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; > + > + tst_resm(TINFO, "total time '%ld' ms", clnt_time); > + > + /* ask server to terminate */ > + client_msg[0] = start_fin_byte; > + int cfd = client_connect_send(client_msg, client_msg_size); > + if (cfd != -1) { > + shutdown(cfd, SHUT_WR); > + SAFE_CLOSE(NULL, cfd); > + } > + /* the script tcp_fastopen_run.sh will remove it */ > + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); > +} > + > +static void client_cleanup(void) > +{ > + void *res = NULL; > + int i; > + for (i = 0; i < threads_num; ++i) > + pthread_join(thread_ids[i], &res); > + > + free(thread_ids); > + > + if (remote_addrinfo) > + freeaddrinfo(remote_addrinfo); > +} > + > + > +static char *make_server_reply(int size) > +{ > + char *send_msg = SAFE_MALLOC(NULL, size); > + memset(send_msg, server_byte, size - 1); > + send_msg[0] = start_byte; > + send_msg[size - 1] = end_byte; > + > + return send_msg; > +} > + > +void *server_fn(void *cfd) > +{ > + int client_fd = (intptr_t) cfd; > + int num_requests = 0, offset = 0; > + > + /* Reply will be constructed from first client request */ > + char *send_msg = NULL; > + int send_msg_size = 0; > + > + char recv_msg[max_msg_len]; > + > + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); > + ssize_t recv_len; > + > + while (1) { > + recv_len = sock_recv_poll(client_fd, recv_msg, > + max_msg_len, &offset); > + > + if (recv_len == 0) { > + break; > + } else > + if (recv_len < 0 || (offset + recv_len) > max_msg_len || > + (recv_msg[0] != start_byte && > + recv_msg[0] != start_fin_byte)) { > + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); > + break; > + } This part of the code looks wrongly indented. Is the else branch needed at all? > + offset += recv_len; > + > + if (recv_msg[offset - 1] != end_byte) { > + tst_resm(TINFO, "msg is not complete, continue recv"); > + continue; > + } > + > + if (recv_msg[0] == start_fin_byte) > + tst_brkm(TBROK, cleanup, "client asks to terminate..."); I still dont like the TBROK on clean exit. Please change it to: cleanup(); tst_exit(); Actually I may be inclined to add a test exit function that would call cleanup first as the cleanup(); tst_exit() is quite common pattern. void tst_exit2(void (*cleanup)(void)); Or something similar would do. Unfortunatelly we cannot change tst_exit() because changing every single usage in tests is not feasible. > + if (verbose) { > + tst_resm_hexd(TINFO, recv_msg, offset, > + "msg recv from sock %d:", client_fd); > + } > + > + /* if we send reply for the first time, construct it here */ > + if (!send_msg) { > + send_msg_size = parse_client_request(recv_msg); > + send_msg = make_server_reply(send_msg_size); > + } > + > + /* > + * It will tell client that server is going > + * to close this connection. > + */ > + if (++num_requests >= server_max_requests) > + send_msg[0] = start_fin_byte; > + > + if (send(client_fd, send_msg, send_msg_size, > + MSG_NOSIGNAL) == -1) { > + tst_resm(TWARN | TERRNO, "Error while sending msg"); > + } else if (verbose) { > + tst_resm_hexd(TINFO, send_msg, send_msg_size, > + "msg sent:"); > + } > + > + offset = 0; > + > + if (num_requests >= server_max_requests) { > + if (verbose) > + tst_resm(TINFO, "Max reqs, close socket"); > + shutdown(client_fd, SHUT_WR); > + break; > + } > + } > + > + free(send_msg); > + SAFE_CLOSE(NULL, client_fd); > + > + return NULL; > +} > + > +static void server_thread_add(intptr_t client_fd) > +{ > + pthread_t id; > + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { > + tst_brkm(TBROK | TERRNO, cleanup, > + "pthread_create failed at %s:%d", __FILE__, __LINE__); > + } > +} > + > +static void server_init(void) > +{ > + struct addrinfo hints; > + memset(&hints, 0, sizeof(struct addrinfo)); > + hints.ai_family = AF_INET; > + hints.ai_socktype = SOCK_STREAM; > + hints.ai_flags = AI_PASSIVE; > + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); > + > + sfd = socket(AF_INET, SOCK_STREAM, 0); > + if (sfd == -1) > + tst_brkm(TBROK, cleanup, "Failed to create a socket"); > + > + tst_resm(TINFO, "assigning a name to the server socket..."); > + if (!local_addrinfo) > + tst_brkm(TBROK, cleanup, "failed to get the address"); > + > + while (bind(sfd, local_addrinfo->ai_addr, > + local_addrinfo->ai_addrlen) == -1) { > + usleep(100000); > + } > + tst_resm(TINFO, "the name assigned"); > + > + freeaddrinfo(local_addrinfo); > + > + const int flag = 1; > + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); > + > + if (fastopen_api == TFO_ENABLED) { > + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, > + sizeof(tfo_queue_size)) == -1) > + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); > + } > + > + listen(sfd, max_queue_len); > + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); > +} > + > +static void server_cleanup(void) > +{ > + SAFE_CLOSE(NULL, sfd); > +} > + > +static void server_run(void) > +{ > + struct sockaddr_in client_addr; > + socklen_t addr_size = sizeof(client_addr); > + pthread_attr_init(&attr); > + > + /* > + * detaching threads allow to reclaim thread's resources > + * once a thread finishes its work. > + */ > + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) > + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); > + > + while (1) { > + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, > + &addr_size); > + if (client_fd == -1) { > + tst_brkm(TBROK, cleanup, "Can't create client socket"); > + continue; Hmm, if I'm not mistaken, the tst_brkm() will abort the test execution, what is the continue needed for? > + } > + > + if (client_addr.sin_family == AF_INET) { > + if (verbose) { > + tst_resm(TINFO, "conn: port '%d', addr '%s'", > + client_addr.sin_port, > + inet_ntoa(client_addr.sin_addr)); > + } > + } > + server_thread_add(client_fd); > + } > +} > + > +static void check_opt(const char *name, char *arg, int *val, int lim) > +{ > + if (arg) { > + if (sscanf(arg, "%i", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", > + name, lim); > + } > +} > + > +static void check_opt_l(const char *name, char *arg, long *val, long lim) > +{ > + if (arg) { > + if (sscanf(arg, "%ld", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) > + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", > + name, lim); > + } > +} > + > +static void setup(int argc, char *argv[]) > +{ > + char *msg; > + msg = parse_opts(argc, argv, options, help); > + if (msg != NULL) > + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); > + > + /* if client num is not set, use num of processors */ > + clients_num = sysconf(_SC_NPROCESSORS_ONLN); > + > + check_opt("a", aarg, &clients_num, 1); > + check_opt("r", rarg, &client_max_requests, 1); > + check_opt("R", Rarg, &server_max_requests, 1); > + check_opt("n", narg, &client_msg_size, 1); > + check_opt("N", Narg, &server_msg_size, 1); > + check_opt("q", qarg, &tfo_queue_size, 1); > + check_opt_l("T", Targ, &wait_timeout, 0L); > + > + if (!force_run) > + tst_require_root(NULL); > + > + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { > + tst_brkm(TCONF, NULL, > + "Test must be run with kernel 3.7 or newer"); > + } > + > + /* check tcp fast open knob */ > + if (!force_run && access(tfo_cfg, F_OK) == -1) > + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); > + > + if (!force_run) { > + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); > + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); > + } > + > + tst_sig(FORK, DEF_HANDLER, cleanup); > + > + tst_resm(TINFO, "TCP %s is using %s TCP API.", > + (tcp_mode == TCP_SERVER) ? "server" : "client", > + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); > + > + switch (tcp_mode) { > + case TCP_SERVER: > + tst_resm(TINFO, "max requests '%d'", > + server_max_requests); > + tcp.init = server_init; > + tcp.run = server_run; > + tcp.cleanup = server_cleanup; > + tfo_bit_num = 2; > + break; > + case TCP_CLIENT: > + tst_resm(TINFO, "connection: %s:%s", > + server_addr, tcp_port); > + tst_resm(TINFO, "client max req: %d", client_max_requests); > + tst_resm(TINFO, "clients num: %d", clients_num); > + tst_resm(TINFO, "client msg size: %d", client_msg_size); > + tst_resm(TINFO, "server msg size: %d", server_msg_size); > + > + tcp.init = client_init; > + tcp.run = client_run; > + tcp.cleanup = client_cleanup; > + tfo_bit_num = 1; > + break; > + } > + > + tfo_support = TFO_ENABLED == tfo_support; > + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { > + int value = (tfo_cfg_value & ~tfo_bit_num) > + | (tfo_support << (tfo_bit_num - 1)); > + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); > + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); > + tfo_cfg_changed = 1; > + } > + > + int reuse_value = 0; > + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); > + if (!reuse_value) { > + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); > + tw_reuse_changed = 1; > + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); > + } > + > + tst_resm(TINFO, "TFO support %s", > + (tfo_support) ? "enabled" : "disabled"); > + > + tcp.init(); > +} > + > +int main(int argc, char *argv[]) > +{ > + setup(argc, argv); > + > + tcp.run(); > + > + cleanup(); > + > + tst_exit(); > +} > diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > new file mode 100755 > index 0000000..f329a0e > --- /dev/null > +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh > @@ -0,0 +1,164 @@ > +#!/bin/bash > + > +# Copyright (c) 2013 Oracle and/or its affiliates. All Rights Reserved. > +# > +# This program is free software; you can redistribute it and/or > +# modify it under the terms of the GNU General Public License as > +# published by the Free Software Foundation; either version 2 of > +# the License, or (at your option) any later version. > +# > +# This program is distributed in the hope that it would be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write the Free Software Foundation, > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA > +# > +# Author: Alexey Kodanev <ale...@or...> > +# > + > +# default command-line options > +user_name="root" > +remote_addr=$RHOST > +use_ssh=0 > +clients_num=2 > +client_requests=2000000 > +max_requests=3 > +server_port=$[ $RANDOM % 28232 + 32768 ] Use $(( )) instead of the non-portable $[ ] > +while getopts :hu:H:sr:p:n:R: opt; do > + case "$opt" in > + h) > + echo "Usage:" > + echo "h help" > + echo "u x server user name" > + echo "H x server hostname or IP address" > + echo "s use ssh to run remote cmds" > + echo "n x num of clients running in parallel" > + echo "r x the number of client requests" > + echo "R x num of requests, after which conn. closed" > + echo "p x server port" > + exit 0 > + ;; > + u) user_name=$OPTARG ;; > + H) remote_addr=$OPTARG ;; > + s) use_ssh=1 ;; > + n) clients_num=$OPTARG ;; > + r) client_requests=$OPTARG ;; > + R) max_requests=$OPTARG ;; > + p) server_port=$OPTARG ;; > + *) > + tst_brkm TBROK NULL "unknown option: $opt" > + exit 2 > + ;; > + esac > +done > + > +run_remote_cmd() > +{ > + tst_resm TINFO "run cmd on $remote_addr: $1" > + > + if [ "$use_ssh" = 1 ]; then > + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" > + else > + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" > + fi > +} > + > +cleanup() > +{ > + rm -f $tfo_result > + run_remote_cmd "pkill -9 tcp_fastopen\$" > +} > + > +read_result_file() > +{ > + if [ -f $tfo_result ]; then > + if [ -r $tfo_result ]; then > + cat $tfo_result > + else > + tst_brkm TBROK NULL "Failed to read result file" > + exit 2 > + fi > + else > + tst_brkm TBROK NULL "Failed to find result file" > + exit 2 > + fi > +} > + > +check_exit_status() > +{ > + if [ "$1" -ne "0" ]; then > + tst_brkm TBROK NULL "Last test has failed" > + exit $1; > + fi > +} > + > +export RC=0 > +export TST_TOTAL=1 > +export TCID="tcp_fastopen" > +export TST_COUNT=0 > + > +tfo_result_ms=0 > +bind_timeout=30 > + > +tst_kvercmp 3 7 0 > +if [ $? -eq 0 ]; then > + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" > + exit 0 > +fi > + > +if [ -z $remote_addr ]; then > + tst_brkm TBROK NULL "you must specify server address" > + exit 2 > +fi > + > +tdir="${LTPROOT}/testcases/bin/" > +tfo_result="${TMPDIR}/tfo_result" > + > +trap "cleanup" EXIT > + > +# Run test without/with TFO > +tcp_api_opt=("-o -O" "") Arrays are unportable bash extension, please do not use them. > +# Time results for without/with TFO test > +vtime=(0 0) > + > +for (( i = 0; i < 2; ++i )); do > + # kill tcp server on remote machine > + run_remote_cmd "pkill -9 tcp_fastopen\$" > + > + sleep 2 > + > + # run tcp server on remote machine > + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ > +${tcp_api_opt[$i]} -g $server_port > /dev/null 2>&1" > + > + sleep $bind_timeout > + > + # run local tcp client > + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ > +-H $remote_addr ${tcp_api_opt[$i]} -g $server_port -d $tfo_result > + > + check_exit_status $? > + > + vtime[$i]=`read_result_file` Here as well. > + if [ -z ${vtime[$i]} -o ${vtime[$i]} -eq "0" ]; then > + tst_brkm TBROK NULL "Last test result isn't valid: ${vtime[$i]}" > + exit 2 > + fi > + server_port=$[ $server_port + 1 ] > +done > + > +tfo_cmp=$[ 100 - (${vtime[1]} * 100) / ${vtime[0]} ] Here as well (the $[]). > +if (( $tfo_cmp < 3 )); then > + tst_resm TFAIL "TFO performance result is $tfo_cmp percent" > + exit 1 > +fi The (( expression )) is bashishm and will not work in most shells. Please use [ $tfo_cmp -lt 3 ] instead. > +tst_resm TPASS "TFO performance result is $tfo_cmp percent" > +exit 0 Otherwise it looks OK, sorry for not catching the bashishms earlier. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2014-01-22 12:24:31
|
Hi! On 12/12/2013 08:39 PM, ch...@su... wrote: > Hi! >> This is a perfomance test for TCP Fast Open (TFO) which is an extension to >> speed up the opening of TCP connections between two endpoints. It reduces >> the number of round time trips (RTT) required in TCP conversations. TFO could >> result in speed improvements of between 4% and 41% in the page load times on >> popular web sites. >> >> The default test scenario simulates an average conversation between a >> web-browser and an application server, so the test results with TFO enabled >> must be at least 3 percent faster. >> >> The test must be run on Linux versions higher then 3.7. (TFO client side >> implemented in Linux 3.6, server side in Linux 3.7). >> >> Signed-off-by: Alexey Kodanev<ale...@or...> >> --- >> testcases/network/tcp_fastopen/.gitignore | 1 + >> testcases/network/tcp_fastopen/Makefile | 24 + >> testcases/network/tcp_fastopen/README | 16 + >> testcases/network/tcp_fastopen/tcp_fastopen.c | 783 ++++++++++++++++++++ >> testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 164 ++++ >> 5 files changed, 988 insertions(+), 0 deletions(-) >> create mode 100644 testcases/network/tcp_fastopen/.gitignore >> create mode 100644 testcases/network/tcp_fastopen/Makefile >> create mode 100644 testcases/network/tcp_fastopen/README >> create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c >> create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh >> >> >> ... >> >> +struct tcp_func { >> + void (*init)(void); >> + void (*run)(void); >> + void (*cleanup)(void); >> +}; >> +static struct tcp_func tcp; >> + >> +#define MAX_THREADS 10000 >> +static pthread_attr_t attr; >> +static pthread_t *thread_ids; >> +static int threads_num; >> + >> +static struct addrinfo *remote_addrinfo; >> +static struct addrinfo *local_addrinfo; >> +static const struct linger clo = { 1, 3 }; >> + >> +static void cleanup(void) >> +{ >> + static int first = 1; >> + if (!first) >> + return; >> + first = 0; > There is still chance for a race condition here. But this could be > easily fixed by pthread_once(). > > static pthread_once_t cleanup_executed = PTHREAD_ONCE_INIT; > > static void do_cleanup(void) > { > /* Do the cleanup here */ > } > > static void cleanup(void) > { > pthread_once(&cleanup_executed, do_cleanup); > } > > Thinking of the problems with LTP library thread safety again. One of > the solutions may be having two tst_res.c implementaions. Note that the > source of thread safety problems is the cleanup function which is > executed by the tst_brkm() and the exitval that is in the test library > too. > > If we did so then we will have tst_res.c we have now and one providing > thread safe implemenation using pthread synchronization primitives > (likely both will be implemented on the top of the common code). > > And if we could switch which one gets linked to a test at the linker > phase depending whether it uses pthreads or not the thread safety > problem will be fixed once for all. > > I will think about this a bit more. As I understand correctly we need to create one more library libltp_mt.a and implement the same but in the thread-safe way, right? So the following part of tst_brk() also requires synchronization, tst_brk_entered in particular. void tst_brk(int ttype, char *fname, void (*func) (void), char *arg_fmt, ...) { ... /* * If no cleanup function was specified, just return to the caller. * Otherwise call the specified function. */ if (func != NULL) { tst_brk_entered++; (*func) (); tst_brk_entered--; } if (tst_brk_entered == 0) tst_exit(); } tst_brk_entered and T_exitval, tst_count can be implemented with gcc atomic bultins. Cleanup function as you suggested will be implemented with pthread_once in a test. Then we can exit with TCONF if a test with libltp thread-safety requirements can't find gcc atomic built-ins. Anyway, I'm thinking that at least for now I won't use tst_res.c in the test, once thread-safe ltplib is done I will make use of it here. > >> ... >> >> +static int client_connect_send(const char *msg, int size) >> +{ >> + int cfd = socket(AF_INET, SOCK_STREAM, 0); >> + const int flag = 1; >> + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); >> + >> + if (cfd == -1) { >> + tst_resm(TWARN | TERRNO, "socket failed at %s:%d", >> + __FILE__, __LINE__); >> + return cfd; >> + } >> + >> + if (fastopen_api == TFO_ENABLED) { >> + /* Replaces connect() + send()/write() */ >> + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, >> + remote_addrinfo->ai_addr, >> + remote_addrinfo->ai_addrlen) != size) { >> + tst_resm(TFAIL | TERRNO, "sendto failed"); >> + SAFE_CLOSE(NULL, cfd); >> + return -1; >> + } >> + } else { >> + /* old TCP API */ >> + if (connect(cfd, remote_addrinfo->ai_addr, >> + remote_addrinfo->ai_addrlen)) { >> + tst_resm(TFAIL | TERRNO, "connect failed"); >> + SAFE_CLOSE(NULL, cfd); >> + return -1; >> + } >> + >> + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { >> + tst_resm(TFAIL | TERRNO, >> + "send failed on sock '%d'", cfd); >> + SAFE_CLOSE(NULL, cfd); >> + return -1; >> + } >> + } >> + >> + return cfd; >> +} >> + >> +void *client_fn(void *arg) >> +{ >> + char buf[server_msg_size]; >> + int cfd, i; >> + >> + /* connect & send requests */ >> + cfd = client_connect_send(client_msg, client_msg_size); >> + if (cfd == -1) >> + return NULL; > So if the connection breaks after the server was started and the client > fails to connect the test will run fine and the resoult would be two > nearly identical runtimes? In that case client will exit with error message and TFAIL exit status (connect() or send() failed). Then tcp_fastopen_run.sh will check client's exit value and will TBROK if the value isn't zero. The bad thing here: it will try to close server and write result file to no purpose. >> ... >> >> + >> +static void make_client_request(void) >> +{ >> + client_msg[0] = start_byte; >> + >> + /* set size for reply */ >> + *(uint16_t *)(client_msg + 1) = htons(server_msg_size); > This will generate unaligned access on some architectures. > > What you need is to set the value byte by byte: > > client_msg[1] = (sever_msg_size>>8) & 0xff; > client_msg[2] = sever_msg_size & 0xff; > >> + client_msg[client_msg_size - 1] = end_byte; >> +} >> + >> +static int parse_client_request(const char *recv_msg) >> +{ >> + int size = ntohs(*(uint16_t *)(recv_msg + 1)); > Here as well, you need to construct the value byte by byte as: > > uint16_t val = htons(((((uint16_t)recv_msg[1])<<8) + recv_msg[2])); OK, but I think it is better to use an union as follows, isn't it? union net_size_field { char bytes[2]; uint16_t value; }; static void make_client_request(void) { ... union net_size_field net_size; net_size.value = htons(server_msg_size); client_msg[1] = net_size.bytes[0]; client_msg[2] = net_size.bytes[1]; .. } static int parse_client_request(const char *recv_msg) { union net_size_field net_size; net_size.bytes[0] = recv_msg[1]; net_size.bytes[1] = recv_msg[2]; int size = ntohs(net_size.value); ... } >> ... >> >> + while (1) { >> + recv_len = sock_recv_poll(client_fd, recv_msg, >> + max_msg_len, &offset); >> + >> + if (recv_len == 0) { >> + break; >> + } else >> + if (recv_len < 0 || (offset + recv_len) > max_msg_len || >> + (recv_msg[0] != start_byte && >> + recv_msg[0] != start_fin_byte)) { >> + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); >> + break; >> + } > This part of the code looks wrongly indented. Is the else branch needed > at all? It isn't needed, I'll fix it. >> ... >> >> + if (recv_msg[offset - 1] != end_byte) { >> + tst_resm(TINFO, "msg is not complete, continue recv"); >> + continue; >> + } >> + >> + if (recv_msg[0] == start_fin_byte) >> + tst_brkm(TBROK, cleanup, "client asks to terminate..."); > I still dont like the TBROK on clean exit. Please change it to: > > cleanup(); > tst_exit(); > > Actually I may be inclined to add a test exit function that would call > cleanup first as the cleanup(); tst_exit() is quite common pattern. > > void tst_exit2(void (*cleanup)(void)); > > Or something similar would do. Unfortunatelly we cannot change > tst_exit() because changing every single usage in tests is not > feasible. OK, I'll replace it with the three functions: tst_resm(TINFO,...) cleanup() tst_exit() Best regards, Alexey |
From: <ch...@su...> - 2014-01-28 14:26:43
|
Hi! > >> +struct tcp_func { > >> + void (*init)(void); > >> + void (*run)(void); > >> + void (*cleanup)(void); > >> +}; > >> +static struct tcp_func tcp; > >> + > >> +#define MAX_THREADS 10000 > >> +static pthread_attr_t attr; > >> +static pthread_t *thread_ids; > >> +static int threads_num; > >> + > >> +static struct addrinfo *remote_addrinfo; > >> +static struct addrinfo *local_addrinfo; > >> +static const struct linger clo = { 1, 3 }; > >> + > >> +static void cleanup(void) > >> +{ > >> + static int first = 1; > >> + if (!first) > >> + return; > >> + first = 0; > > There is still chance for a race condition here. But this could be > > easily fixed by pthread_once(). > > > > static pthread_once_t cleanup_executed = PTHREAD_ONCE_INIT; > > > > static void do_cleanup(void) > > { > > /* Do the cleanup here */ > > } > > > > static void cleanup(void) > > { > > pthread_once(&cleanup_executed, do_cleanup); > > } > > > > Thinking of the problems with LTP library thread safety again. One of > > the solutions may be having two tst_res.c implementaions. Note that the > > source of thread safety problems is the cleanup function which is > > executed by the tst_brkm() and the exitval that is in the test library > > too. > > > > If we did so then we will have tst_res.c we have now and one providing > > thread safe implemenation using pthread synchronization primitives > > (likely both will be implemented on the top of the common code). > > > > And if we could switch which one gets linked to a test at the linker > > phase depending whether it uses pthreads or not the thread safety > > problem will be fixed once for all. > > > > I will think about this a bit more. > As I understand correctly we need to create one more library libltp_mt.a > and implement the same but in the thread-safe way, right? That is one of the posible solutions. I would create the thread safe library on the top of the one we have now and created set of calls with _r suffix (as libc does with strerror(), ctime(), ...). Then we can use these when needed and link the the test against both -lltp and -lltp_mt (or whatever it gets called). That would include all the tst_res() functions and maybe some other, we can add these once they. The open question is the callback() because we either need to redirect all calls that gets the callback pointer as an parameter or create the callback indirection (with pthread once) in the test. I would rather see the second variant. What we can do is a macro that will decleare the pthread once etc given the name of the real callback and the name of the resulting function. > So the following part of tst_brk() also requires synchronization, > tst_brk_entered in particular. > > void tst_brk(int ttype, char *fname, void (*func) (void), char *arg_fmt, > ...) > { > ... > > /* > * If no cleanup function was specified, just return to the caller. > * Otherwise call the specified function. > */ > if (func != NULL) { > tst_brk_entered++; > (*func) (); > tst_brk_entered--; > } > if (tst_brk_entered == 0) > tst_exit(); > > } > > tst_brk_entered and T_exitval, tst_count can be implemented with gcc > atomic bultins. Cleanup function as you suggested will be implemented > with pthread_once in a test. Then we can exit with TCONF if a test with > libltp thread-safety requirements can't find gcc atomic built-ins. This part is a hack for being able to call safe macros from within the test cleanup(). All safe macros call tst_brkm() if something has failed which will exit the cleanup() prematurely. But this does not work well or reliably. I would vote for removing this part and forbiding calling safe macros from within the test cleanup(), people tend to pass the cleanup function pointer to these without thinking which creates infinite recursion and the test is killed when stack is filled up... > Anyway, I'm thinking that at least for now I won't use tst_res.c in the > test, once thread-safe ltplib is done I will make use of it here. OK. But we should fix the LTP lib as well. > >> + > >> +static void make_client_request(void) > >> +{ > >> + client_msg[0] = start_byte; > >> + > >> + /* set size for reply */ > >> + *(uint16_t *)(client_msg + 1) = htons(server_msg_size); > > This will generate unaligned access on some architectures. > > > > What you need is to set the value byte by byte: > > > > client_msg[1] = (sever_msg_size>>8) & 0xff; > > client_msg[2] = sever_msg_size & 0xff; > > > >> + client_msg[client_msg_size - 1] = end_byte; > >> +} > >> + > >> +static int parse_client_request(const char *recv_msg) > >> +{ > >> + int size = ntohs(*(uint16_t *)(recv_msg + 1)); > > Here as well, you need to construct the value byte by byte as: > > > > uint16_t val = htons(((((uint16_t)recv_msg[1])<<8) + recv_msg[2])); > OK, but I think it is better to use an union as follows, isn't it? > > union net_size_field { > char bytes[2]; > uint16_t value; > }; > > static void make_client_request(void) > { > ... > union net_size_field net_size; > net_size.value = htons(server_msg_size); > client_msg[1] = net_size.bytes[0]; > client_msg[2] = net_size.bytes[1]; > .. > } > > static int parse_client_request(const char *recv_msg) > { > union net_size_field net_size; > net_size.bytes[0] = recv_msg[1]; > net_size.bytes[1] = recv_msg[2]; > int size = ntohs(net_size.value); > ... > } > That should work as well. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2014-02-07 13:42:26
|
Hi! On 01/28/2014 06:26 PM, ch...@su... wrote: > Hi! > > That is one of the posible solutions. > > I would create the thread safe library on the top of the one we have now > and created set of calls with _r suffix (as libc does with strerror(), > ctime(), ...). Then we can use these when needed and link the the test > against both -lltp and -lltp_mt (or whatever it gets called). That would > include all the tst_res() functions and maybe some other, we can add > these once they. > > The open question is the callback() because we either need to redirect > all calls that gets the callback pointer as an parameter or create the > callback indirection (with pthread once) in the test. I would rather see > the second variant. What we can do is a macro that will decleare the > pthread once etc given the name of the real callback and the name of the > resulting function. > OK, that would work. I'm sending first two patches to separate common code from tst_res.c and to declare callback() in multi-threaded tests. Thanks, Alexey |
From: Alexey K. <ale...@or...> - 2014-02-05 10:43:30
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/tcp_fastopen.c | 810 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 173 +++++ 4 files changed, 1008 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..aa60bb0 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread -lrt + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..56cd674 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,810 @@ +/* + * Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/sysinfo.h> +#include <sys/poll.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static const int client_byte = 0x43; +static const int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * LTP lib is not thread-safe yet and server's threads are detached + * so make use of mutex protected variable server_exit_err to get errno + * from the threads + */ +int server_exit_err; +pthread_mutex_t mutex; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"N:", NULL, &Narg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; +static int threads_num; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static pthread_once_t cleanup_executed = PTHREAD_ONCE_INIT; + +static void do_cleanup(void) +{ + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} + +static void cleanup(void) +{ + pthread_once(&cleanup_executed, do_cleanup); +} + +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + break; + } + + if (ret == 0) { + errno = ETIME; + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(fd, buf + *offset, + buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + return len; +} + +static int client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + errno = 0; + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + errno = ENOMSG; + break; + } + offset += len; + if (buf[offset - 1] != end_byte) + continue; + /* recv last msg, close socket */ + if (buf[0] == start_fin_byte) + break; + return 0; + } + + shutdown(*fd, SHUT_WR); + close(*fd); + *fd = -1; + return (errno) ? -1 : 0; +} + +static int client_connect_send(const char *msg, int size) +{ + int cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (cfd == -1) + return cfd; + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != size) { + close(cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + close(cfd); + return -1; + } + + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { + close(cfd); + return -1; + } + } + + return cfd; +} + +void *client_fn(void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + intptr_t err = 0; + + /* connect & send requests */ + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + goto out; + } + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + close(cfd); + + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + + continue; + + } else if (ret > 0) { + err = EMSGSIZE; + break; + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + err = ECOMM; + break; + } + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + } + + if (cfd != -1) + close(cfd); + +out: + return (void *) err; +} + +union net_size_field { + char bytes[2]; + uint16_t value; +}; + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* set size for reply */ + union net_size_field net_size; + net_size.value = htons(server_msg_size); + client_msg[1] = net_size.bytes[0]; + client_msg[2] = net_size.bytes[1]; + + client_msg[client_msg_size - 1] = end_byte; +} + +static int parse_client_request(const char *msg) +{ + union net_size_field net_size; + net_size.bytes[0] = msg[1]; + net_size.bytes[1] = msg[2]; + int size = ntohs(net_size.value); + if (size < 2 || size > max_msg_len) + return -1; + + return size; +} + +static struct timespec tv_client_start; +static struct timespec tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + ++threads_num; + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + int i; + for (i = 0; i < clients_num; ++i) { + pthread_join(thread_ids[i], &res); + if (res) { + tst_brkm(TBROK, cleanup, "client[%d] failed: %s", + i, strerror((intptr_t)res)); + } + } + + threads_num = 0; + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time); + + /* ask server to terminate */ + client_msg[0] = start_fin_byte; + int cfd = client_connect_send(client_msg, client_msg_size); + if (cfd != -1) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); +} + +static void client_cleanup(void) +{ + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + +static char *make_server_reply(int size) +{ + char *send_msg = SAFE_MALLOC(NULL, size); + memset(send_msg, server_byte, size - 1); + send_msg[0] = start_byte; + send_msg[size - 1] = end_byte; + return send_msg; +} + +static void set_server_exit_err(int err) +{ + pthread_mutex_lock(&mutex); + server_exit_err = err; + pthread_mutex_unlock(&mutex); +} + +static int get_server_exit_err() +{ + int err; + pthread_mutex_lock(&mutex); + err = server_exit_err; + pthread_mutex_unlock(&mutex); + return err; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL; + int send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + recv_len = sock_recv_poll(client_fd, recv_msg, + max_msg_len, &offset); + + if (recv_len == 0) + break; + + if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + set_server_exit_err(ENOMSG); + goto out; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + /* msg is not complete, continue recv */ + continue; + } + + /* client asks to terminate */ + if (recv_msg[0] == start_fin_byte) + goto out; + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + send_msg_size = parse_client_request(recv_msg); + if (send_msg_size < 0) { + set_server_exit_err(EPROTO); + goto out; + } + send_msg = make_server_reply(send_msg_size); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) { + set_server_exit_err(errno); + goto out; + } + + offset = 0; + + if (num_requests >= server_max_requests) { + /* max reqs, close socket */ + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + close(client_fd); + return NULL; + +out: + free(send_msg); + close(client_fd); + cleanup(); + tst_exit(); +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + usleep(100000); + } + tst_resm(TINFO, "the name assigned"); + + freeaddrinfo(local_addrinfo); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + pthread_mutex_init(&mutex, NULL); + server_exit_err = 0; + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); + int err = get_server_exit_err(); + tst_resm((err) ? TFAIL : TPASS, "%s", + (err) ? strerror(err) : "server test completed"); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) + tst_brkm(TBROK, cleanup, "Can't create client socket"); + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client num is not set, use num of processors */ + clients_num = sysconf(_SC_NPROCESSORS_ONLN); + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + check_opt("N", Narg, &server_msg_size, 1); + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..bffc420 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,173 @@ +#!/bin/sh + +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 +server_port=$(( $RANDOM % 28232 + 32768 )) + +export RC=0 +export TST_TOTAL=1 +export TCID="tcp_fastopen" +export TST_COUNT=0 +bind_timeout=15 +tdir="${LTPROOT}/testcases/bin/" +tfo_result="${TMPDIR}/tfo_result" + + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + echo "p x server port" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + p) server_port=$OPTARG ;; + *) + tst_brkm TBROK NULL "unknown option: $opt" + exit 2 + ;; + esac +done + +run_remote_cmd() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + + if [ "$use_ssh" = 1 ]; then + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" \ +> /dev/null 2>&1 + else + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" \ +> /dev/null 2>&1 + fi + if [ $? -ne 0 ]; then + tst_brkm TBROK NULL "No route to host $RHOST" + exit 2 + fi +} + +cleanup() +{ + run_remote_cmd "pkill -9 tcp_fastopen\$" + rm -f $tfo_result + sleep 1 +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK NULL "Failed to read result file" + exit 2 + fi + else + tst_brkm TBROK NULL "Failed to find result file" + exit 2 + fi +} + +check_exit_status() +{ + if [ "$1" -ne 0 ]; then + tst_brkm TBROK NULL "Last test has failed" + exit $1; + fi +} + +run_client_server() +{ + local tfo_opt=$1 + # kill tcp server on remote machine + run_remote_cmd "pkill -9 tcp_fastopen\$" + sleep 2 + + # run tcp server on remote machine + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ +$tfo_opt -g $server_port > /dev/null 2>&1" + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr $tfo_opt -g $server_port -d $tfo_result + check_exit_status $? + + run_time=`read_result_file` + + if [ -z "$run_time" -o "$run_time" -eq 0 ]; then + tst_brkm TBROK NULL "Last test result isn't valid: $run_time" + exit 2 + fi + server_port=$(( $server_port + 1 )) +} + +if [ "`id -u`" -ne 0 ]; then + tst_brkm TCONF NULL "Test must be run as root" + exit 0 +fi + +tst_kvercmp 3 7 0 +if [ $? -eq 0 ]; then + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" + exit 0 +fi + +if [ -z $remote_addr ]; then + tst_brkm TBROK NULL "you must specify server address" + exit 2 +fi + +trap "cleanup" EXIT + +run_client_server "-o -O" +time_tfo_off=$run_time + +run_client_server +time_tfo_on=$run_time + +tfo_cmp=$(( 100 - ($time_tfo_on * 100) / $time_tfo_off )) + +if [ "$tfo_cmp" -lt 3 ]; then + tst_resm TFAIL "TFO performance result is '$tfo_cmp' percent" + exit 1 +fi + +tst_resm TPASS "TFO performance result is $tfo_cmp percent" +exit 0 -- 1.7.1 |
From: Alexey K. <ale...@or...> - 2014-02-19 11:29:39
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- runtest/network_stress.tcp | 2 + testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/tcp_fastopen.c | 777 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 173 +++++ 5 files changed, 977 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/runtest/network_stress.tcp b/runtest/network_stress.tcp index 7206b3a..1361d6b 100644 --- a/runtest/network_stress.tcp +++ b/runtest/network_stress.tcp @@ -331,3 +331,5 @@ tcp6-multi-diffnic11 tcp6-multi-diffnic11 tcp6-multi-diffnic12 tcp6-multi-diffnic12 tcp6-multi-diffnic13 tcp6-multi-diffnic13 tcp6-multi-diffnic14 tcp6-multi-diffnic14 + +tcp_fastopen_run.sh tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..aa60bb0 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread -lrt + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..650e127 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,777 @@ +/* + * Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <pthread.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netdb.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <poll.h> +#include <time.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static const int client_byte = 0x43; +static const int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"N:", NULL, &Narg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void do_cleanup(void) +{ + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} +DECLARE_ONCE_FN(cleanup, do_cleanup) + +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + break; + } + + if (ret == 0) { + errno = ETIME; + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(fd, buf + *offset, + buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + return len; +} + +static int client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + errno = 0; + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + errno = ENOMSG; + break; + } + offset += len; + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + /* recv last msg, close socket */ + if (buf[0] == start_fin_byte) + break; + return 0; + } + + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(cleanup, *fd); + *fd = -1; + return (errno) ? -1 : 0; +} + +static int client_connect_send(const char *msg, int size) +{ + int cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (cfd == -1) + return cfd; + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } + + return cfd; +} + +void *client_fn(LTP_ATTRIBUTE_UNUSED void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + intptr_t err = 0; + + /* connect & send requests */ + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + goto out; + } + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + + continue; + + } else if (ret > 0) { + err = EMSGSIZE; + break; + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + err = ECOMM; + break; + } + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + } + + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + +out: + return (void *) err; +} + +union net_size_field { + char bytes[2]; + uint16_t value; +}; + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* set size for reply */ + union net_size_field net_size; + net_size.value = htons(server_msg_size); + client_msg[1] = net_size.bytes[0]; + client_msg[2] = net_size.bytes[1]; + + client_msg[client_msg_size - 1] = end_byte; +} + +static int parse_client_request(const char *msg) +{ + union net_size_field net_size; + net_size.bytes[0] = msg[1]; + net_size.bytes[1] = msg[2]; + int size = ntohs(net_size.value); + if (size < 2 || size > max_msg_len) + return -1; + + return size; +} + +static struct timespec tv_client_start; +static struct timespec tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + int i; + for (i = 0; i < clients_num; ++i) { + pthread_join(thread_ids[i], &res); + if (res) { + tst_brkm(TBROK, cleanup, "client[%d] failed: %s", + i, strerror((intptr_t)res)); + } + } + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time); + + /* ask server to terminate */ + client_msg[0] = start_fin_byte; + int cfd = client_connect_send(client_msg, client_msg_size); + if (cfd != -1) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); +} + +static void client_cleanup(void) +{ + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + +static char *make_server_reply(int size) +{ + char *send_msg = SAFE_MALLOC(NULL, size); + memset(send_msg, server_byte, size - 1); + send_msg[0] = start_byte; + send_msg[size - 1] = end_byte; + return send_msg; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL; + int send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + recv_len = sock_recv_poll(client_fd, recv_msg, + max_msg_len, &offset); + + if (recv_len == 0) + break; + + if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); + goto out; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + /* msg is not complete, continue recv */ + continue; + } + + /* client asks to terminate */ + if (recv_msg[0] == start_fin_byte) + goto out; + + if (verbose) { + tst_resm_hexd(TINFO, recv_msg, offset, + "msg recv from sock %d:", client_fd); + } + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + send_msg_size = parse_client_request(recv_msg); + if (send_msg_size < 0) { + tst_resm(TFAIL, "wrong msg size '%d'", + send_msg_size); + goto out; + } + send_msg = make_server_reply(send_msg_size); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) { + tst_resm(TFAIL | TERRNO, "send failed"); + goto out; + } + + offset = 0; + + if (num_requests >= server_max_requests) { + /* max reqs, close socket */ + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + return NULL; + +out: + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + cleanup(); + tst_exit(); +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + usleep(100000); + } + tst_resm(TINFO, "the name assigned"); + + freeaddrinfo(local_addrinfo); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) + tst_brkm(TBROK, cleanup, "Can't create client socket"); + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client_num is not set, use num of processors */ + clients_num = sysconf(_SC_NPROCESSORS_ONLN); + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + check_opt("N", Narg, &server_msg_size, 1); + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..0549e17 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,173 @@ +#!/bin/sh + +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 +server_port=$(( $RANDOM % 28232 + 32768 )) + +export RC=0 +export TST_TOTAL=1 +export TCID="tcp_fastopen" +export TST_COUNT=0 +bind_timeout=15 +tdir="${LTPROOT}/testcases/bin/" +tfo_result="${TMPDIR}/tfo_result" + + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + echo "p x server port" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + p) server_port=$OPTARG ;; + *) + tst_brkm TBROK NULL "unknown option: $opt" + exit 2 + ;; + esac +done + +run_remote_cmd() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + + if [ "$use_ssh" -eq 1 ]; then + ssh -n -f $user_name@$remote_addr "sh -c 'nohup $1 &'" \ +> /dev/null 2>&1 + else + rsh -n -l $user_name $remote_addr "sh -c 'nohup $1 &'" \ +> /dev/null 2>&1 + fi + if [ $? -ne 0 ]; then + tst_brkm TBROK NULL "No route to host $RHOST" + exit 2 + fi +} + +cleanup() +{ + run_remote_cmd "pkill -9 tcp_fastopen\$" + rm -f $tfo_result + sleep 1 +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK NULL "Failed to read result file" + exit 2 + fi + else + tst_brkm TBROK NULL "Failed to find result file" + exit 2 + fi +} + +check_exit_status() +{ + if [ "$1" -ne 0 ]; then + tst_brkm TBROK NULL "Last test has failed" + exit $1; + fi +} + +run_client_server() +{ + local tfo_opt=$1 + # kill tcp server on remote machine + run_remote_cmd "pkill -9 tcp_fastopen\$" + sleep 2 + + # run tcp server on remote machine + run_remote_cmd "${tdir}tcp_fastopen -R $max_requests \ +$tfo_opt -g $server_port > /dev/null 2>&1" + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr $tfo_opt -g $server_port -d $tfo_result + check_exit_status $? + + run_time=`read_result_file` + + if [ -z "$run_time" -o "$run_time" -eq 0 ]; then + tst_brkm TBROK NULL "Last test result isn't valid: $run_time" + exit 2 + fi + server_port=$(( $server_port + 1 )) +} + +if [ "`id -u`" -ne 0 ]; then + tst_brkm TCONF NULL "Test must be run as root" + exit 0 +fi + +tst_kvercmp 3 7 0 +if [ $? -eq 0 ]; then + tst_brkm TCONF NULL "test must be run with kernel 3.7 or newer" + exit 0 +fi + +if [ -z $remote_addr ]; then + tst_brkm TBROK NULL "you must specify server address" + exit 2 +fi + +trap "cleanup" EXIT + +run_client_server "-o -O" +time_tfo_off=$run_time + +run_client_server +time_tfo_on=$run_time + +tfo_cmp=$(( 100 - ($time_tfo_on * 100) / $time_tfo_off )) + +if [ "$tfo_cmp" -lt 3 ]; then + tst_resm TFAIL "TFO performance result is '$tfo_cmp' percent" + exit 1 +fi + +tst_resm TPASS "TFO performance result is $tfo_cmp percent" +exit 0 -- 1.7.1 |
From: <ch...@su...> - 2014-03-13 14:48:24
|
Hi! > This is a perfomance test for TCP Fast Open (TFO) which is an extension to > speed up the opening of TCP connections between two endpoints. It reduces > the number of round time trips (RTT) required in TCP conversations. TFO could > result in speed improvements of between 4% and 41% in the page load times on > popular web sites. > > The default test scenario simulates an average conversation between a > web-browser and an application server, so the test results with TFO enabled > must be at least 3 percent faster. > > The test must be run on Linux versions higher then 3.7. (TFO client side > implemented in Linux 3.6, server side in Linux 3.7). This version looks good. I would be happier if the shell part was ported to the new test.sh but I can live with it as it is now. And this patch is blocked by the thread safety patch for the tst_res.c. I guess I will commit it next week, it has been on the ML long enough. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2014-03-17 10:28:55
|
Hi! On 03/13/2014 06:48 PM, ch...@su... wrote: > Hi! >> This is a perfomance test for TCP Fast Open (TFO) which is an extension to >> speed up the opening of TCP connections between two endpoints. It reduces >> the number of round time trips (RTT) required in TCP conversations. TFO could >> result in speed improvements of between 4% and 41% in the page load times on >> popular web sites. >> >> The default test scenario simulates an average conversation between a >> web-browser and an application server, so the test results with TFO enabled >> must be at least 3 percent faster. >> >> The test must be run on Linux versions higher then 3.7. (TFO client side >> implemented in Linux 3.6, server side in Linux 3.7). > This version looks good. I would be happier if the shell part was ported > to the new test.sh but I can live with it as it is now. > > And this patch is blocked by the thread safety patch for the tst_res.c. > I guess I will commit it next week, it has been on the ML long enough. > OK, thanks. I can do porting later along with the fix which I would like to add to the test: there is recently added tst_get_unused_port() function, but how can I use it in the script, should I add it to tools/apicmds? Thanks, Alexey |
From: <ch...@su...> - 2014-03-17 13:18:25
|
Hi! > > This version looks good. I would be happier if the shell part was ported > > to the new test.sh but I can live with it as it is now. > > > > And this patch is blocked by the thread safety patch for the tst_res.c. > > I guess I will commit it next week, it has been on the ML long enough. > > > OK, thanks. I can do porting later along with the fix which I would like > to add to the test: there is recently added tst_get_unused_port() > function, but how can I use it in the script, should I add it to > tools/apicmds? Yes please. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2014-03-17 15:57:49
|
On 03/17/2014 05:18 PM, ch...@su... wrote: > Hi! >>> This version looks good. I would be happier if the shell part was ported >>> to the new test.sh but I can live with it as it is now. >>> >>> And this patch is blocked by the thread safety patch for the tst_res.c. >>> I guess I will commit it next week, it has been on the ML long enough. >>> >> OK, thanks. I can do porting later along with the fix which I would like >> to add to the test: there is recently added tst_get_unused_port() >> function, but how can I use it in the script, should I add it to >> tools/apicmds? > Yes please. > OK, so what could be the parameters for the command? The function takes family and type, as a result command could be as follows: $ tst_get_unused_port ipv4 stream then we need to parse strings and pass the right ones to the C function. or we can set ids and explain the meaning in the help: $ tst_get_unused_port 1 0 Which one is better? |
From: <ch...@su...> - 2014-03-17 16:01:43
|
Hi! > >>> This version looks good. I would be happier if the shell part was ported > >>> to the new test.sh but I can live with it as it is now. > >>> > >>> And this patch is blocked by the thread safety patch for the tst_res.c. > >>> I guess I will commit it next week, it has been on the ML long enough. > >>> > >> OK, thanks. I can do porting later along with the fix which I would like > >> to add to the test: there is recently added tst_get_unused_port() > >> function, but how can I use it in the script, should I add it to > >> tools/apicmds? > > Yes please. > > > OK, so what could be the parameters for the command? > The function takes family and type, as a result command could be as follows: > $ tst_get_unused_port ipv4 stream > then we need to parse strings and pass the right ones to the C function. > > or we can set ids and explain the meaning in the help: > $ tst_get_unused_port 1 0 > > Which one is better? I would vote for the strings. You can easily decleare an array of structures with string-value paris and search in these. It's not that much of work and the shell code would be much easier to read. -- Cyril Hrubis ch...@su... |
From: Alexey K. <ale...@or...> - 2014-03-17 16:08:49
|
On 03/17/2014 08:01 PM, ch...@su... wrote: > Hi! >>>>> This version looks good. I would be happier if the shell part was ported >>>>> to the new test.sh but I can live with it as it is now. >>>>> >>>>> And this patch is blocked by the thread safety patch for the tst_res.c. >>>>> I guess I will commit it next week, it has been on the ML long enough. >>>>> >>>> OK, thanks. I can do porting later along with the fix which I would like >>>> to add to the test: there is recently added tst_get_unused_port() >>>> function, but how can I use it in the script, should I add it to >>>> tools/apicmds? >>> Yes please. >>> >> OK, so what could be the parameters for the command? >> The function takes family and type, as a result command could be as follows: >> $ tst_get_unused_port ipv4 stream >> then we need to parse strings and pass the right ones to the C function. >> >> or we can set ids and explain the meaning in the help: >> $ tst_get_unused_port 1 0 >> >> Which one is better? > I would vote for the strings. You can easily decleare an array of > structures with string-value paris and search in these. It's not that > much of work and the shell code would be much easier to read. > OK |
From: Alexey K. <ale...@or...> - 2014-03-19 14:11:05
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- v6: Make use of new LTP library: test.sh Use tst_get_unused_port instead of choosing random port fix: replace DECLARE_ONCE_FN with TST_DECLARE_* runtest/network_stress.tcp | 2 + testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/tcp_fastopen.c | 779 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 172 +++++ 5 files changed, 978 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/runtest/network_stress.tcp b/runtest/network_stress.tcp index 7206b3a..4bd2cbb 100644 --- a/runtest/network_stress.tcp +++ b/runtest/network_stress.tcp @@ -331,3 +331,5 @@ tcp6-multi-diffnic11 tcp6-multi-diffnic11 tcp6-multi-diffnic12 tcp6-multi-diffnic12 tcp6-multi-diffnic13 tcp6-multi-diffnic13 tcp6-multi-diffnic14 tcp6-multi-diffnic14 + +tcp_fastopen_run tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..aa60bb0 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread -lrt + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..4dba26b --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,779 @@ +/* + * Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <pthread.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netdb.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <poll.h> +#include <time.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static const int client_byte = 0x43; +static const int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"N:", NULL, &Narg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void do_cleanup(void) +{ + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} +TST_DECLARE_ONCE_FN(cleanup, do_cleanup) + +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + break; + } + + if (ret == 0) { + errno = ETIME; + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(fd, buf + *offset, + buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + return len; +} + +static int client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + errno = 0; + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); + + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + if (!errno) + errno = ENOMSG; + break; + } + offset += len; + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + /* recv last msg, close socket */ + if (buf[0] == start_fin_byte) + break; + return 0; + } + + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(cleanup, *fd); + *fd = -1; + return (errno) ? -1 : 0; +} + +static int client_connect_send(const char *msg, int size) +{ + int cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (cfd == -1) + return cfd; + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } + + return cfd; +} + +void *client_fn(LTP_ATTRIBUTE_UNUSED void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + intptr_t err = 0; + + /* connect & send requests */ + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + goto out; + } + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + + continue; + + } else if (ret > 0) { + err = EMSGSIZE; + break; + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + err = ECOMM; + break; + } + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + } + + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + +out: + return (void *) err; +} + +union net_size_field { + char bytes[2]; + uint16_t value; +}; + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* set size for reply */ + union net_size_field net_size; + net_size.value = htons(server_msg_size); + client_msg[1] = net_size.bytes[0]; + client_msg[2] = net_size.bytes[1]; + + client_msg[client_msg_size - 1] = end_byte; +} + +static int parse_client_request(const char *msg) +{ + union net_size_field net_size; + net_size.bytes[0] = msg[1]; + net_size.bytes[1] = msg[2]; + int size = ntohs(net_size.value); + if (size < 2 || size > max_msg_len) + return -1; + + return size; +} + +static struct timespec tv_client_start; +static struct timespec tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + int i; + for (i = 0; i < clients_num; ++i) { + pthread_join(thread_ids[i], &res); + if (res) { + tst_brkm(TBROK, cleanup, "client[%d] failed: %s", + i, strerror((intptr_t)res)); + } + } + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time); + + /* ask server to terminate */ + client_msg[0] = start_fin_byte; + int cfd = client_connect_send(client_msg, client_msg_size); + if (cfd != -1) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); +} + +static void client_cleanup(void) +{ + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + +static char *make_server_reply(int size) +{ + char *send_msg = SAFE_MALLOC(NULL, size); + memset(send_msg, server_byte, size - 1); + send_msg[0] = start_byte; + send_msg[size - 1] = end_byte; + return send_msg; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL; + int send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + recv_len = sock_recv_poll(client_fd, recv_msg, + max_msg_len, &offset); + + if (recv_len == 0) + break; + + if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); + goto out; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + /* msg is not complete, continue recv */ + continue; + } + + /* client asks to terminate */ + if (recv_msg[0] == start_fin_byte) + goto out; + + if (verbose) { + tst_resm_hexd(TINFO, recv_msg, offset, + "msg recv from sock %d:", client_fd); + } + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + send_msg_size = parse_client_request(recv_msg); + if (send_msg_size < 0) { + tst_resm(TFAIL, "wrong msg size '%d'", + send_msg_size); + goto out; + } + send_msg = make_server_reply(send_msg_size); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) { + tst_resm(TFAIL | TERRNO, "send failed"); + goto out; + } + + offset = 0; + + if (num_requests >= server_max_requests) { + /* max reqs, close socket */ + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + return NULL; + +out: + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + cleanup(); + tst_exit(); +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + usleep(100000); + } + tst_resm(TINFO, "the name assigned"); + + freeaddrinfo(local_addrinfo); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) + tst_brkm(TBROK, cleanup, "Can't create client socket"); + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client_num is not set, use num of processors */ + clients_num = sysconf(_SC_NPROCESSORS_ONLN); + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + check_opt("N", Narg, &server_msg_size, 1); + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..8c02564 --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,172 @@ +#!/bin/sh + +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 + +export TST_TOTAL=1 +export TCID="tcp_fastopen" + +. test.sh + +bind_timeout=5 +tdir="${LTPROOT}/testcases/bin/" +tfo_result="${TMPDIR}/tfo_result" + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) use_ssh=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + *) + tst_brkm TBROK "unknown option: $opt" + ;; + esac +done + +tst_rhost_run() +{ + if [ "$use_ssh" -eq 1 ]; then + ssh -n -f $user_name@$RHOST "sh -c 'nohup $@ &'" \ +> /dev/null 2>&1 + else + rsh -n -l $user_name $RHOST "sh -c 'nohup $@ &'" \ +> /dev/null 2>&1 + fi + return $? +} + +get_rhost_unused_port() +{ + local get_port="TCID=$TCID TST_COUNT=1 TST_TOTAL=1 \ +${tdir}tst_get_unused_port ipv4 stream" + + if [ "$use_ssh" -eq 1 ]; then + port=`ssh -n -f $user_name@$RHOST "sh -c '$get_port'"` + else + port=`rsh -n -l $user_name $RHOST "sh -c '$get_port'"` + fi + [ $? -ne 0 ] && tst_brkm TBROK "get_rhost_unused_port failed" + echo $port +} + +cleanup() +{ + tst_resm TINFO "cleanup..." + tst_rhost_run "pkill -9 tcp_fastopen\$" + rm -f $tfo_result + sleep 1 +} + +TST_CLEANUP="cleanup" +trap "tst_brkm TBROK 'test interrupted'" SIGINT + +safe_rhost_run() +{ + tst_resm TINFO "run cmd on $remote_addr: $1" + tst_rhost_run $@ + [ $? -ne 0 ] && tst_brkm TBROK "No route to host $RHOST" +} + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK "Failed to read result file" + fi + else + tst_brkm TBROK "Failed to find result file" + fi +} + +check_exit_status() +{ + [ "$1" -ne 0 ] && tst_brkm TBROK "Last test has failed" +} + +run_client_server() +{ + local tfo_opt=$1 + # kill tcp server on remote machine + safe_rhost_run "pkill -9 tcp_fastopen\$" + sleep 2 + + local server_port=`get_rhost_unused_port` + + # run tcp server on remote machine + safe_rhost_run "${tdir}tcp_fastopen -R $max_requests \ +$tfo_opt -g $server_port > /dev/null 2>&1" + sleep $bind_timeout + + # run local tcp client + ${tdir}tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr $tfo_opt -g $server_port -d $tfo_result + check_exit_status $? + + run_time=`read_result_file` + + [ -z "$run_time" -o "$run_time" -eq 0 ] && \ + tst_brkm TBROK "Last test result isn't valid: $run_time" +} + +tst_require_root + +tst_kvercmp 3 7 0 +[ $? -eq 0 ] && tst_brkm TCONF "test must be run with kernel 3.7 or newer" + +[ -z $remote_addr ] && tst_brkm TBROK "you must specify server address" + +run_client_server "-o -O" +time_tfo_off=$run_time + +run_client_server +time_tfo_on=$run_time + +tfo_cmp=$(( 100 - ($time_tfo_on * 100) / $time_tfo_off )) + +if [ "$tfo_cmp" -lt 3 ]; then + tst_resm TFAIL "TFO performance result is '$tfo_cmp' percent" +else + tst_resm TPASS "TFO performance result is $tfo_cmp percent" +fi + +tst_exit -- 1.7.1 |
From: Alexey K. <ale...@or...> - 2014-03-26 14:48:54
|
This is a perfomance test for TCP Fast Open (TFO) which is an extension to speed up the opening of TCP connections between two endpoints. It reduces the number of round time trips (RTT) required in TCP conversations. TFO could result in speed improvements of between 4% and 41% in the page load times on popular web sites. The default test scenario simulates an average conversation between a web-browser and an application server, so the test results with TFO enabled must be at least 3 percent faster. The test must be run on Linux versions higher then 3.7. (TFO client side implemented in Linux 3.6, server side in Linux 3.7). Signed-off-by: Alexey Kodanev <ale...@or...> --- v7: Use test_net.sh Style and portability fixes in tcp_fastopen_run.sh v6: Make use of new LTP library: test.sh Use tst_get_unused_port instead of choosing random port fix: replace DECLARE_ONCE_FN with TST_DECLARE_* runtest/network_stress.tcp | 2 + testcases/network/tcp_fastopen/.gitignore | 1 + testcases/network/tcp_fastopen/Makefile | 24 + testcases/network/tcp_fastopen/tcp_fastopen.c | 779 ++++++++++++++++++++ testcases/network/tcp_fastopen/tcp_fastopen_run.sh | 130 ++++ 5 files changed, 936 insertions(+), 0 deletions(-) create mode 100644 testcases/network/tcp_fastopen/.gitignore create mode 100644 testcases/network/tcp_fastopen/Makefile create mode 100644 testcases/network/tcp_fastopen/tcp_fastopen.c create mode 100755 testcases/network/tcp_fastopen/tcp_fastopen_run.sh diff --git a/runtest/network_stress.tcp b/runtest/network_stress.tcp index 7206b3a..4bd2cbb 100644 --- a/runtest/network_stress.tcp +++ b/runtest/network_stress.tcp @@ -331,3 +331,5 @@ tcp6-multi-diffnic11 tcp6-multi-diffnic11 tcp6-multi-diffnic12 tcp6-multi-diffnic12 tcp6-multi-diffnic13 tcp6-multi-diffnic13 tcp6-multi-diffnic14 tcp6-multi-diffnic14 + +tcp_fastopen_run tcp_fastopen_run.sh diff --git a/testcases/network/tcp_fastopen/.gitignore b/testcases/network/tcp_fastopen/.gitignore new file mode 100644 index 0000000..f321cf0 --- /dev/null +++ b/testcases/network/tcp_fastopen/.gitignore @@ -0,0 +1 @@ +/tcp_fastopen diff --git a/testcases/network/tcp_fastopen/Makefile b/testcases/network/tcp_fastopen/Makefile new file mode 100644 index 0000000..aa60bb0 --- /dev/null +++ b/testcases/network/tcp_fastopen/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +top_srcdir ?= ../../.. + +include $(top_srcdir)/include/mk/testcases.mk + +INSTALL_TARGETS := tcp_fastopen_run.sh +LDLIBS += -lpthread -lrt + +include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/testcases/network/tcp_fastopen/tcp_fastopen.c b/testcases/network/tcp_fastopen/tcp_fastopen.c new file mode 100644 index 0000000..4dba26b --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen.c @@ -0,0 +1,779 @@ +/* + * Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Author: Alexey Kodanev <ale...@or...> + * + */ + +#include <pthread.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netdb.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <poll.h> +#include <time.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> + +#include "test.h" +#include "usctest.h" +#include "safe_macros.h" + +char *TCID = "tcp_fastopen"; + +static const int max_msg_len = 1500; + +/* TCP server requiers */ +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* TCP client requiers */ +#ifndef MSG_FASTOPEN +#define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ +#endif + +enum { + TCP_SERVER = 0, + TCP_CLIENT, +}; +static int tcp_mode; + +enum { + TFO_ENABLED = 0, + TFO_DISABLED, +}; +static int tfo_support; +static int fastopen_api; + +static const char tfo_cfg[] = "/proc/sys/net/ipv4/tcp_fastopen"; +static const char tcp_tw_reuse[] = "/proc/sys/net/ipv4/tcp_tw_reuse"; +static int tw_reuse_changed; +static int tfo_cfg_value; +static int tfo_bit_num; +static int tfo_cfg_changed; +static int tfo_queue_size = 100; +static int max_queue_len = 100; +static const int client_byte = 0x43; +static const int server_byte = 0x53; +static const int start_byte = 0x24; +static const int start_fin_byte = 0x25; +static const int end_byte = 0x0a; +static int client_msg_size = 32; +static int server_msg_size = 128; +static char *client_msg; +static char *server_msg; + +/* + * The number of requests from client after + * which server has to close the connection. + */ +static int server_max_requests = 3; +static int client_max_requests = 10; +static int clients_num = 2; +static char *tcp_port = "61000"; +static char *server_addr = "localhost"; +/* server socket */ +static int sfd; + +/* how long a client must wait for the server's reply, microsec */ +static long wait_timeout = 10000000; + +/* in the end test will save time result in this file */ +static char *rpath = "./tfo_result"; + +static int force_run; +static int verbose; + +static char *narg, *Narg, *qarg, *rarg, *Rarg, *aarg, *Targ; + +static const option_t options[] = { + /* server params */ + {"R:", NULL, &Rarg}, + {"q:", NULL, &qarg}, + + /* client params */ + {"H:", NULL, &server_addr}, + {"a:", NULL, &aarg}, + {"n:", NULL, &narg}, + {"N:", NULL, &Narg}, + {"T:", NULL, &Targ}, + {"r:", NULL, &rarg}, + {"d:", NULL, &rpath}, + + /* common */ + {"g:", NULL, &tcp_port}, + {"F", &force_run, NULL}, + {"l", &tcp_mode, NULL}, + {"o", &fastopen_api, NULL}, + {"O", &tfo_support, NULL}, + {"v", &verbose, NULL}, + {NULL, NULL, NULL} +}; + +static void help(void) +{ + printf("\n -F Force to run\n"); + printf(" -v Verbose\n"); + printf(" -o Use old TCP API, default is new TCP API\n"); + printf(" -O TFO support is off, default is on\n"); + printf(" -l Become TCP Client, default is TCP server\n"); + printf(" -g x x - server port, default is %s\n", tcp_port); + + printf("\n Client:\n"); + printf(" -H x x - server name or ip address, default is '%s'\n", + server_addr); + printf(" -a x x - num of clients running in parallel\n"); + printf(" -r x x - num of client requests\n"); + printf(" -n x Client message size, max msg size is '%d'\n", + max_msg_len); + printf(" -N x Server message size, max msg size is '%d'\n", + max_msg_len); + printf(" -T x Reply timeout, default is '%ld' (microsec)\n", + wait_timeout); + printf(" -d x x is a path to the file where results are saved\n"); + + printf("\n Server:\n"); + printf(" -R x x - num of requests, after which conn. closed\n"); + printf(" -q x x - server's limit on the queue of TFO requests\n"); +} + +/* common structure for TCP server and TCP client */ +struct tcp_func { + void (*init)(void); + void (*run)(void); + void (*cleanup)(void); +}; +static struct tcp_func tcp; + +#define MAX_THREADS 10000 +static pthread_attr_t attr; +static pthread_t *thread_ids; + +static struct addrinfo *remote_addrinfo; +static struct addrinfo *local_addrinfo; +static const struct linger clo = { 1, 3 }; + +static void do_cleanup(void) +{ + free(client_msg); + free(server_msg); + + tcp.cleanup(); + + if (tfo_cfg_changed) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tfo_cfg_value &= ~tfo_bit_num; + tfo_cfg_value |= !tfo_support << (tfo_bit_num - 1); + tst_resm(TINFO, "unset '%s' back to '%d'", + tfo_cfg, tfo_cfg_value); + SAFE_FILE_PRINTF(NULL, tfo_cfg, "%d", tfo_cfg_value); + } + + if (tw_reuse_changed) { + SAFE_FILE_PRINTF(NULL, tcp_tw_reuse, "0"); + tst_resm(TINFO, "unset '%s' back to '0'", tcp_tw_reuse); + } + TEST_CLEANUP; +} +TST_DECLARE_ONCE_FN(cleanup, do_cleanup) + +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int len = -1; + while (1) { + errno = 0; + int ret = poll(&pfd, 1, wait_timeout / 1000); + if (ret == -1) { + if (errno == EINTR) + continue; + break; + } + + if (ret == 0) { + errno = ETIME; + break; + } + + if (ret != 1 || !(pfd.revents & POLLIN)) + break; + + errno = 0; + len = recv(fd, buf + *offset, + buf_size - *offset, MSG_DONTWAIT); + + if (len == -1 && errno == EINTR) + continue; + else + break; + } + + return len; +} + +static int client_recv(int *fd, char *buf) +{ + int len, offset = 0; + + while (1) { + errno = 0; + len = sock_recv_poll(*fd, buf, server_msg_size, &offset); + + /* socket closed or msg is not valid */ + if (len < 1 || (offset + len) > server_msg_size || + (buf[0] != start_byte && buf[0] != start_fin_byte)) { + if (!errno) + errno = ENOMSG; + break; + } + offset += len; + if (buf[offset - 1] != end_byte) + continue; + + if (verbose) { + tst_resm_hexd(TINFO, buf, offset, + "msg recv from sock %d:", *fd); + } + + /* recv last msg, close socket */ + if (buf[0] == start_fin_byte) + break; + return 0; + } + + shutdown(*fd, SHUT_WR); + SAFE_CLOSE(cleanup, *fd); + *fd = -1; + return (errno) ? -1 : 0; +} + +static int client_connect_send(const char *msg, int size) +{ + int cfd = socket(AF_INET, SOCK_STREAM, 0); + const int flag = 1; + setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (cfd == -1) + return cfd; + + if (fastopen_api == TFO_ENABLED) { + /* Replaces connect() + send()/write() */ + if (sendto(cfd, msg, size, MSG_FASTOPEN | MSG_NOSIGNAL, + remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen) != size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } else { + /* old TCP API */ + if (connect(cfd, remote_addrinfo->ai_addr, + remote_addrinfo->ai_addrlen)) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + + if (send(cfd, msg, size, MSG_NOSIGNAL) != client_msg_size) { + SAFE_CLOSE(cleanup, cfd); + return -1; + } + } + + return cfd; +} + +void *client_fn(LTP_ATTRIBUTE_UNUSED void *arg) +{ + char buf[server_msg_size]; + int cfd, i; + intptr_t err = 0; + + /* connect & send requests */ + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + goto out; + } + + for (i = 1; i < client_max_requests; ++i) { + + /* check connection, it can be closed */ + int ret = 0; + if (cfd != -1) + ret = recv(cfd, buf, 1, MSG_DONTWAIT); + + if (ret == 0) { + /* try to reconnect and send */ + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + + cfd = client_connect_send(client_msg, client_msg_size); + if (cfd == -1) { + err = errno; + goto out; + } + + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + + continue; + + } else if (ret > 0) { + err = EMSGSIZE; + break; + } + + if (verbose) { + tst_resm_hexd(TINFO, client_msg, client_msg_size, + "try to send msg[%d]", i); + } + + if (send(cfd, client_msg, client_msg_size, + MSG_NOSIGNAL) != client_msg_size) { + err = ECOMM; + break; + } + if (client_recv(&cfd, buf)) { + err = errno; + break; + } + } + + if (cfd != -1) + SAFE_CLOSE(cleanup, cfd); + +out: + return (void *) err; +} + +union net_size_field { + char bytes[2]; + uint16_t value; +}; + +static void make_client_request(void) +{ + client_msg[0] = start_byte; + + /* set size for reply */ + union net_size_field net_size; + net_size.value = htons(server_msg_size); + client_msg[1] = net_size.bytes[0]; + client_msg[2] = net_size.bytes[1]; + + client_msg[client_msg_size - 1] = end_byte; +} + +static int parse_client_request(const char *msg) +{ + union net_size_field net_size; + net_size.bytes[0] = msg[1]; + net_size.bytes[1] = msg[2]; + int size = ntohs(net_size.value); + if (size < 2 || size > max_msg_len) + return -1; + + return size; +} + +static struct timespec tv_client_start; +static struct timespec tv_client_end; + +static void client_init(void) +{ + if (clients_num >= MAX_THREADS) { + tst_brkm(TBROK, cleanup, + "Unexpected num of clients '%d'", + clients_num); + } + + thread_ids = SAFE_MALLOC(NULL, sizeof(pthread_t) * clients_num); + + client_msg = SAFE_MALLOC(NULL, client_msg_size); + memset(client_msg, client_byte, client_msg_size); + + make_client_request(); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(server_addr, tcp_port, &hints, &remote_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_start); + int i; + for (i = 0; i < clients_num; ++i) { + if (pthread_create(&thread_ids[i], 0, client_fn, NULL) != 0) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", + __FILE__, __LINE__); + } + } +} + +static void client_run(void) +{ + void *res = NULL; + long clnt_time = 0; + int i; + for (i = 0; i < clients_num; ++i) { + pthread_join(thread_ids[i], &res); + if (res) { + tst_brkm(TBROK, cleanup, "client[%d] failed: %s", + i, strerror((intptr_t)res)); + } + } + + clock_gettime(CLOCK_MONOTONIC_RAW, &tv_client_end); + clnt_time = (tv_client_end.tv_sec - tv_client_start.tv_sec) * 1000 + + (tv_client_end.tv_nsec - tv_client_start.tv_nsec) / 1000000; + + tst_resm(TINFO, "total time '%ld' ms", clnt_time); + + /* ask server to terminate */ + client_msg[0] = start_fin_byte; + int cfd = client_connect_send(client_msg, client_msg_size); + if (cfd != -1) { + shutdown(cfd, SHUT_WR); + SAFE_CLOSE(NULL, cfd); + } + /* the script tcp_fastopen_run.sh will remove it */ + SAFE_FILE_PRINTF(cleanup, rpath, "%ld", clnt_time); +} + +static void client_cleanup(void) +{ + free(thread_ids); + + if (remote_addrinfo) + freeaddrinfo(remote_addrinfo); +} + +static char *make_server_reply(int size) +{ + char *send_msg = SAFE_MALLOC(NULL, size); + memset(send_msg, server_byte, size - 1); + send_msg[0] = start_byte; + send_msg[size - 1] = end_byte; + return send_msg; +} + +void *server_fn(void *cfd) +{ + int client_fd = (intptr_t) cfd; + int num_requests = 0, offset = 0; + + /* Reply will be constructed from first client request */ + char *send_msg = NULL; + int send_msg_size = 0; + + char recv_msg[max_msg_len]; + + setsockopt(client_fd, SOL_SOCKET, SO_LINGER, &clo, sizeof(clo)); + ssize_t recv_len; + + while (1) { + recv_len = sock_recv_poll(client_fd, recv_msg, + max_msg_len, &offset); + + if (recv_len == 0) + break; + + if (recv_len < 0 || (offset + recv_len) > max_msg_len || + (recv_msg[0] != start_byte && + recv_msg[0] != start_fin_byte)) { + tst_resm(TFAIL, "recv failed, sock '%d'", client_fd); + goto out; + } + + offset += recv_len; + + if (recv_msg[offset - 1] != end_byte) { + /* msg is not complete, continue recv */ + continue; + } + + /* client asks to terminate */ + if (recv_msg[0] == start_fin_byte) + goto out; + + if (verbose) { + tst_resm_hexd(TINFO, recv_msg, offset, + "msg recv from sock %d:", client_fd); + } + + /* if we send reply for the first time, construct it here */ + if (!send_msg) { + send_msg_size = parse_client_request(recv_msg); + if (send_msg_size < 0) { + tst_resm(TFAIL, "wrong msg size '%d'", + send_msg_size); + goto out; + } + send_msg = make_server_reply(send_msg_size); + } + + /* + * It will tell client that server is going + * to close this connection. + */ + if (++num_requests >= server_max_requests) + send_msg[0] = start_fin_byte; + + if (send(client_fd, send_msg, send_msg_size, + MSG_NOSIGNAL) == -1) { + tst_resm(TFAIL | TERRNO, "send failed"); + goto out; + } + + offset = 0; + + if (num_requests >= server_max_requests) { + /* max reqs, close socket */ + shutdown(client_fd, SHUT_WR); + break; + } + } + + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + return NULL; + +out: + free(send_msg); + SAFE_CLOSE(cleanup, client_fd); + cleanup(); + tst_exit(); +} + +static void server_thread_add(intptr_t client_fd) +{ + pthread_t id; + if (pthread_create(&id, &attr, server_fn, (void *) client_fd)) { + tst_brkm(TBROK | TERRNO, cleanup, + "pthread_create failed at %s:%d", __FILE__, __LINE__); + } +} + +static void server_init(void) +{ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(NULL, tcp_port, &hints, &local_addrinfo) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "getaddrinfo failed"); + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if (sfd == -1) + tst_brkm(TBROK, cleanup, "Failed to create a socket"); + + tst_resm(TINFO, "assigning a name to the server socket..."); + if (!local_addrinfo) + tst_brkm(TBROK, cleanup, "failed to get the address"); + + while (bind(sfd, local_addrinfo->ai_addr, + local_addrinfo->ai_addrlen) == -1) { + usleep(100000); + } + tst_resm(TINFO, "the name assigned"); + + freeaddrinfo(local_addrinfo); + + const int flag = 1; + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + + if (fastopen_api == TFO_ENABLED) { + if (setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN, &tfo_queue_size, + sizeof(tfo_queue_size)) == -1) + tst_brkm(TBROK, cleanup, "Can't set TFO sock. options"); + } + + listen(sfd, max_queue_len); + tst_resm(TINFO, "Listen on the socket '%d', port '%s'", sfd, tcp_port); +} + +static void server_cleanup(void) +{ + SAFE_CLOSE(NULL, sfd); +} + +static void server_run(void) +{ + struct sockaddr_in client_addr; + socklen_t addr_size = sizeof(client_addr); + pthread_attr_init(&attr); + + /* + * detaching threads allow to reclaim thread's resources + * once a thread finishes its work. + */ + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + tst_brkm(TBROK | TERRNO, cleanup, "setdetachstate failed"); + + while (1) { + int client_fd = accept(sfd, (struct sockaddr *) &client_addr, + &addr_size); + if (client_fd == -1) + tst_brkm(TBROK, cleanup, "Can't create client socket"); + + if (client_addr.sin_family == AF_INET) { + if (verbose) { + tst_resm(TINFO, "conn: port '%d', addr '%s'", + client_addr.sin_port, + inet_ntoa(client_addr.sin_addr)); + } + } + server_thread_add(client_fd); + } +} + +static void check_opt(const char *name, char *arg, int *val, int lim) +{ + if (arg) { + if (sscanf(arg, "%i", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", + name, lim); + } +} + +static void check_opt_l(const char *name, char *arg, long *val, long lim) +{ + if (arg) { + if (sscanf(arg, "%ld", val) != 1) + tst_brkm(TBROK, NULL, "-%s option arg is not a number", + name); + if (clients_num < lim) + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", + name, lim); + } +} + +static void setup(int argc, char *argv[]) +{ + char *msg; + msg = parse_opts(argc, argv, options, help); + if (msg != NULL) + tst_brkm(TBROK, NULL, "OPTION PARSING ERROR - %s", msg); + + /* if client_num is not set, use num of processors */ + clients_num = sysconf(_SC_NPROCESSORS_ONLN); + + check_opt("a", aarg, &clients_num, 1); + check_opt("r", rarg, &client_max_requests, 1); + check_opt("R", Rarg, &server_max_requests, 1); + check_opt("n", narg, &client_msg_size, 1); + check_opt("N", Narg, &server_msg_size, 1); + check_opt("q", qarg, &tfo_queue_size, 1); + check_opt_l("T", Targ, &wait_timeout, 0L); + + if (!force_run) + tst_require_root(NULL); + + if (!force_run && tst_kvercmp(3, 7, 0) < 0) { + tst_brkm(TCONF, NULL, + "Test must be run with kernel 3.7 or newer"); + } + + /* check tcp fast open knob */ + if (!force_run && access(tfo_cfg, F_OK) == -1) + tst_brkm(TCONF, NULL, "Failed to find '%s'", tfo_cfg); + + if (!force_run) { + SAFE_FILE_SCANF(NULL, tfo_cfg, "%d", &tfo_cfg_value); + tst_resm(TINFO, "'%s' is %d", tfo_cfg, tfo_cfg_value); + } + + tst_sig(FORK, DEF_HANDLER, cleanup); + + tst_resm(TINFO, "TCP %s is using %s TCP API.", + (tcp_mode == TCP_SERVER) ? "server" : "client", + (fastopen_api == TFO_ENABLED) ? "Fastopen" : "old"); + + switch (tcp_mode) { + case TCP_SERVER: + tst_resm(TINFO, "max requests '%d'", + server_max_requests); + tcp.init = server_init; + tcp.run = server_run; + tcp.cleanup = server_cleanup; + tfo_bit_num = 2; + break; + case TCP_CLIENT: + tst_resm(TINFO, "connection: %s:%s", + server_addr, tcp_port); + tst_resm(TINFO, "client max req: %d", client_max_requests); + tst_resm(TINFO, "clients num: %d", clients_num); + tst_resm(TINFO, "client msg size: %d", client_msg_size); + tst_resm(TINFO, "server msg size: %d", server_msg_size); + + tcp.init = client_init; + tcp.run = client_run; + tcp.cleanup = client_cleanup; + tfo_bit_num = 1; + break; + } + + tfo_support = TFO_ENABLED == tfo_support; + if (((tfo_cfg_value & tfo_bit_num) == tfo_bit_num) != tfo_support) { + int value = (tfo_cfg_value & ~tfo_bit_num) + | (tfo_support << (tfo_bit_num - 1)); + tst_resm(TINFO, "set '%s' to '%d'", tfo_cfg, value); + SAFE_FILE_PRINTF(cleanup, tfo_cfg, "%d", value); + tfo_cfg_changed = 1; + } + + int reuse_value = 0; + SAFE_FILE_SCANF(cleanup, tcp_tw_reuse, "%d", &reuse_value); + if (!reuse_value) { + SAFE_FILE_PRINTF(cleanup, tcp_tw_reuse, "1"); + tw_reuse_changed = 1; + tst_resm(TINFO, "set '%s' to '1'", tcp_tw_reuse); + } + + tst_resm(TINFO, "TFO support %s", + (tfo_support) ? "enabled" : "disabled"); + + tcp.init(); +} + +int main(int argc, char *argv[]) +{ + setup(argc, argv); + + tcp.run(); + + cleanup(); + + tst_exit(); +} diff --git a/testcases/network/tcp_fastopen/tcp_fastopen_run.sh b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh new file mode 100755 index 0000000..eab8cfd --- /dev/null +++ b/testcases/network/tcp_fastopen/tcp_fastopen_run.sh @@ -0,0 +1,130 @@ +#!/bin/sh + +# Copyright (c) 2014 Oracle and/or its affiliates. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it would be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Alexey Kodanev <ale...@or...> +# + +# default command-line options +user_name="root" +remote_addr=$RHOST +use_ssh=0 +clients_num=2 +client_requests=2000000 +max_requests=3 + +TST_TOTAL=1 +TCID="tcp_fastopen" + +. test_net.sh + +bind_timeout=5 +tfo_result="${TMPDIR}/tfo_result" + +while getopts :hu:H:sr:p:n:R: opt; do + case "$opt" in + h) + echo "Usage:" + echo "h help" + echo "u x server user name" + echo "H x server hostname or IP address" + echo "s use ssh to run remote cmds" + echo "n x num of clients running in parallel" + echo "r x the number of client requests" + echo "R x num of requests, after which conn. closed" + exit 0 + ;; + u) user_name=$OPTARG ;; + H) remote_addr=$OPTARG ;; + s) export TST_USE_SSH=1 ;; + n) clients_num=$OPTARG ;; + r) client_requests=$OPTARG ;; + R) max_requests=$OPTARG ;; + *) + tst_brkm TBROK "unknown option: $opt" + ;; + esac +done + +cleanup() +{ + tst_resm TINFO "cleanup..." + tst_rhost_run -c "pkill -9 tcp_fastopen\$" + rm -f $tfo_result +} + +TST_CLEANUP="cleanup" +trap "tst_brkm TBROK 'test interrupted'" INT + +read_result_file() +{ + if [ -f $tfo_result ]; then + if [ -r $tfo_result ]; then + cat $tfo_result + else + tst_brkm TBROK "Failed to read result file" + fi + else + tst_brkm TBROK "Failed to find result file" + fi +} + +run_client_server() +{ + # kill tcp server on remote machine + tst_rhost_run -c "pkill -9 tcp_fastopen\$" + + port=$(tst_rhost_run -c "tst_get_unused_port ipv4 stream") + [ $? -ne 0 ] && tst_brkm TBROK "failed to get unused port" + + # run tcp server on remote machine + tst_rhost_run -s -b -c "tcp_fastopen -R $max_requests $1 -g $port" + sleep $bind_timeout + + # run local tcp client + tcp_fastopen -a $clients_num -r $client_requests -l \ +-H $remote_addr $1 -g $port -d $tfo_result + [ "$?" -ne 0 ] && tst_brkm TBROK "Last test has failed" + + run_time=$(read_result_file) + + [ -z "$run_time" -o "$run_time" -eq 0 ] && \ + tst_brkm TBROK "Last test result isn't valid: $run_time" +} + +tst_require_root + +tst_kvercmp 3 7 0 +[ $? -eq 0 ] && tst_brkm TCONF "test must be run with kernel 3.7 or newer" + +[ -z $remote_addr ] && tst_brkm TBROK "you must specify server address" + +run_client_server "-o -O" +time_tfo_off=$run_time + +run_client_server +time_tfo_on=$run_time + +tfo_cmp=$(( 100 - ($time_tfo_on * 100) / $time_tfo_off )) + +if [ "$tfo_cmp" -lt 3 ]; then + tst_resm TFAIL "TFO performance result is '$tfo_cmp' percent" +else + tst_resm TPASS "TFO performance result is '$tfo_cmp' percent" +fi + +tst_exit -- 1.7.1 |
From: <ch...@su...> - 2014-04-10 15:40:28
|
Hi! > +static int sock_recv_poll(int fd, char *buf, int buf_size, int *offset) > +{ > + struct pollfd pfd; > + pfd.fd = fd; > + pfd.events = POLLIN; > + int len = -1; > + while (1) { > + errno = 0; > + int ret = poll(&pfd, 1, wait_timeout / 1000); > + if (ret == -1) { > + if (errno == EINTR) > + continue; > + break; > + } > + > + if (ret == 0) { > + errno = ETIME; > + break; > + } > + > + if (ret != 1 || !(pfd.revents & POLLIN)) > + break; > + > + errno = 0; > + len = recv(fd, buf + *offset, > + buf_size - *offset, MSG_DONTWAIT); > + > + if (len == -1 && errno == EINTR) > + continue; > + else > + break; > + } > + > + return len; > +} You pass the offset as a pointer but do not modify the value. Forgot to change it when the code around changed? ... > +static void check_opt(const char *name, char *arg, int *val, int lim) > +{ > + if (arg) { > + if (sscanf(arg, "%i", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) ^ *val > + tst_brkm(TBROK, NULL, "-%s option arg is less than %d", > + name, lim); > + } > +} > + > +static void check_opt_l(const char *name, char *arg, long *val, long lim) > +{ > + if (arg) { > + if (sscanf(arg, "%ld", val) != 1) > + tst_brkm(TBROK, NULL, "-%s option arg is not a number", > + name); > + if (clients_num < lim) ^ *val > + tst_brkm(TBROK, NULL, "-%s option arg is less than %ld", > + name, lim); > + } > +} > + The rest looks fine. -- Cyril Hrubis ch...@su... |