Changes - j0ke.net Open Build Service

@@ -1,10 +0,0 @@ ---- src/glb_pool.c.orig 2009-04-19 21:18:55.000000000 +0200 -+++ src/glb_pool.c 2009-04-19 21:22:57.000000000 +0200 -@@ -31,6 +31,7 @@ - #include "glb_time.h" - #include "glb_log.h" - #include "glb_pool.h" -+#include "glb_pool_stats.h" - - extern bool glb_verbose; -

@@ -1,6 +1,47 @@ +2009-06-07 alex +- Fixed: + * Off-by-one bug which threw control thread into a tight loop when using FIFO + control method. + * listener thread was still using select(). + Version 0.7.2 + +2009-04-18 alex +- Fixed: + * control thread was still using select() to listen to requests (FD_SETSIZE + descriptor limit). + * reverted "fix" which opened control socket by default for security and + usability considerations. + Version 0.7.1. + +2009-04-16 alex +- Added machine parsable stats output on 'getstat' command +- Fixed: + * control socket was not opened by default + * usage display if weight was not 0 or 1 + * real numbers were not accepted for destination weight + Version 0.7.0. + +2009-04-15 alex +- Refactored to use poll/epoll API instead of select(). This allows for more + than FD_SETSIZE/2 concurrent connections and is considerably faster in case + of epoll. + +2009-03-12 alex +- Cosmetic changes to the output: usage now ranges strictly from 0.0 + (unused) to 1.0 (overloaded). Weight 0.0 automatically means usage 1.0, + before that it was 'nan'. Version 0.6.3. + +2009-02-10 alex +- Fixed non-working destination failover, failed destinations are retried + every 2 seconds. Version 0.6.2. + +2009-01-07 alex +- Fixed silly destination list parsing bug. Version 0.6.1. + 2008-09-14 alex - Fixed a problem with default port when adding destinations during runtime. - Changed from comma-separated destination list to space-separated destination list, hence bumped minor version. + 2008-07-15 alex - Initial release.

@@ -19,8 +19,12 @@ - supports server "draining", i.e. does not allocate new connections to server, but does not kill existing ones, waiting for them to end gracefully. - - glb is multithreaded, so it can utilize multiple CPU cores. In fact even - on a single core CPU two threads may lower latencies a bit. + - can use epoll API provided by Linux version 2.6 and higher for ultimate + routing performance. + - glb is multithreaded, so it can utilize multiple CPU cores. Also, if your + OS does not support epoll API, consider using several threads even on a + single clore machine as it will lessen poll() overhead proportionally and + can improve overall performance by a factor of 2 or more. COMMAND LINE OPTIONS: ===================== @@ -49,12 +53,12 @@ OK $ echo "192.168.0.2:3307:5" | nc -q 1 127.0.0.1 4444 OK -$ echo "getstat" | nc -q 1 127.0.0.1 4444 +$ echo "getinfo" | nc -q 1 127.0.0.1 4444 Router: ---------------------------------------------------- - Address : weight usage conns -192.168.0.1:3307 : 5 0.00 0 -192.168.0.2:3307 : 5 0.00 0 + Address : weight usage conns +192.168.0.1:3307 : 5.000 0.000 0 +192.168.0.2:3307 : 5.000 0.000 0 ---------------------------------------------------- Destinations: 2, total connections: 0 @@ -73,3 +77,36 @@ HOSTNAME1,HOSTNAME2,HOSTNAME3. In that case incoming port number will be used for PORT value and 1 will be used for WEIGHT value. + +PERFORMANCE STATISTICS: +======================= +GLB allows to query raw performance statistics through control socket using +'getstat' command. The client can use these data to obtain useful information, +e.g. average number of reads per poll() call. + +$ echo "getstat" | nc -q 1 127.0.0.1 4444 +in: 6930 out: 102728 recv: 109658 / 45 send: 109658 / 45 conns: 0 / 4 poll: 45 / 0 / 45 elapsed: 1.03428 + +Statistics line consists of fields separated by spaces for ease of parsing in +scripts. A few description fields are added to assist in human reading. Value +fields are all even and go as follows: + + 2 - number of bytes received on incoming interface (client requests) + 4 - number of bytes sent from incoming inteface (server responses) + 6 - number of bytes passed through recv() call + 8 - number of recv() calls + 10 - number of bytes passed through send() call (should be equal to p.6) + 12 - number of send() calls + 14 - number of created connections + 16 - number of concurrent connections + 18 - number of read-ready file descriptors returned by poll()/epoll_wait() + 20 - number of write-ready file descriptors returned by poll()/epoll_wait() + 22 - number of times poll()/epoll_wait() triggered + 24 - time elapsed since last statistics report (seconds) + +All values except for 16 and 24 are totals accumulated since the last report. +In order to obtain some variable rate it must be divided by the elapsed time. +On 32-bit architectures the values are stored in 4-byte integers and can +overflow if too much time elapsed, so the first statistics report in the series +may need to be discarded. +

@@ -19,6 +19,9 @@ /* Define to 1 if you have the `memset' function. */ #undef HAVE_MEMSET +/* Define to 1 if you have the <poll.h> header file. */ +#undef HAVE_POLL_H + /* Define to 1 if your system has a GNU libc compatible `realloc' function, and to 0 otherwise. */ #undef HAVE_REALLOC @@ -53,6 +56,9 @@ /* Define to 1 if you have the `strtol' function. */ #undef HAVE_STRTOL +/* Define to 1 if you have the <sys/epoll.h> header file. */ +#undef HAVE_SYS_EPOLL_H + /* Define to 1 if you have the <sys/stat.h> header file. */ #undef HAVE_SYS_STAT_H

@@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.61 for glb 0.6.0. +# Generated by GNU Autoconf 2.61 for glb 0.7.2. # # Report bugs to <info@codership.com>. # @@ -574,8 +574,8 @@ # Identity of this package. PACKAGE_NAME='glb' PACKAGE_TARNAME='glb' -PACKAGE_VERSION='0.6.0' -PACKAGE_STRING='glb 0.6.0' +PACKAGE_VERSION='0.7.2' +PACKAGE_STRING='glb 0.7.2' PACKAGE_BUGREPORT='info@codership.com' ac_unique_file="src/glb_main.c" @@ -686,10 +686,10 @@ AMTAR am__tar am__untar -ENABLE_DEBUG_TRUE -ENABLE_DEBUG_FALSE ENABLE_SPLICE_TRUE ENABLE_SPLICE_FALSE +ENABLE_DEBUG_TRUE +ENABLE_DEBUG_FALSE ENABLE_STATS_TRUE ENABLE_STATS_FALSE CXX @@ -1234,7 +1234,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures glb 0.6.0 to adapt to many kinds of systems. +\`configure' configures glb 0.7.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1305,15 +1305,18 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of glb 0.6.0:";; + short | recursive ) echo "Configuration of glb 0.7.2:";; esac cat <<\_ACEOF Optional Features: --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-debug enable debugging code [default=disabled] + --enable-poll use poll() for polling (default: use epoll() if + available) + --enable-splice use splice() [default=disabled] + --enable-debug enable debugging code [default=disabled] --enable-stats use extensive pool statistics [default=disabled] --disable-dependency-tracking speeds up one-time build --enable-dependency-tracking do not reject slow dependency extractors @@ -1394,7 +1397,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -glb configure 0.6.0 +glb configure 0.7.2 generated by GNU Autoconf 2.61 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1408,7 +1411,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by glb $as_me 0.6.0, which was +It was created by glb $as_me 0.7.2, which was generated by GNU Autoconf 2.61. Invocation command line was $ $0 $@ @@ -2228,7 +2231,7 @@ # Define the identity of the package. PACKAGE='glb' - VERSION='0.6.0' + VERSION='0.7.2' cat >>confdefs.h <<_ACEOF @@ -2377,27 +2380,12 @@ #AM_GNU_GETTEXT #AM_GNU_GETTEXT_VERSION(0.17) -# Check for debug -# Check whether --enable-debug was given. -if test "${enable_debug+set}" = set; then - enableval=$enable_debug; -else - enable_debug="no" -fi - -if test "$enable_debug" == "yes" -then - CFLAGS="-g -O1 -fno-inline " -else - CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} - CFLAGS="$CFLAGS -DNDEBUG " -fi - if test "$enable_debug" != "no"; then - ENABLE_DEBUG_TRUE= - ENABLE_DEBUG_FALSE='#' +# Check for poll method +# Check whether --enable-poll was given. +if test "${enable_poll+set}" = set; then + enableval=$enable_poll; else - ENABLE_DEBUG_TRUE='#' - ENABLE_DEBUG_FALSE= + enable_poll="no" fi @@ -2422,10 +2410,34 @@ fi +# Check for debug +# Check whether --enable-debug was given. +if test "${enable_debug+set}" = set; then + enableval=$enable_debug; +else + enable_debug="no" +fi + +if test "$enable_debug" == "yes" +then + CFLAGS="-g -O0 -fno-inline " +else + CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} + CFLAGS="$CFLAGS -DNDEBUG " +fi + if test "$enable_debug" != "no"; then + ENABLE_DEBUG_TRUE= + ENABLE_DEBUG_FALSE='#' +else + ENABLE_DEBUG_TRUE='#' + ENABLE_DEBUG_FALSE= +fi + + # Check for stats -# Check whether --enable-splice was given. -if test "${enable_splice+set}" = set; then - enableval=$enable_splice; +# Check whether --enable-stats was given. +if test "${enable_stats+set}" = set; then + enableval=$enable_stats; else enable_stats="no" fi @@ -4008,7 +4020,6 @@ #AC_PROG_LIBTOOL -# FIXME: Replace `main' with a function in `-lpthread': { echo "$as_me:$LINENO: checking for pthread_testcancel in -lpthread" >&5 @@ -6129,7 +6140,7 @@ -for ac_func in gettimeofday memset strcasecmp strdup strncasecmp strtol +for ac_func in gettimeofday memset strdup strcasecmp strncasecmp strtol do as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` { echo "$as_me:$LINENO: checking for $ac_func" >&5 @@ -6222,6 +6233,492 @@ fi done + +# Check for poll()/epoll() + +for ac_header in poll.h +do +as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh` +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + { echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } +else + # Is the header compilable? +{ echo "$as_me:$LINENO: checking $ac_header usability" >&5 +echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6; } + +# Is the header present? +{ echo "$as_me:$LINENO: checking $ac_header presence" >&5 +echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <$ac_header> +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5 +echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: $ac_header: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## --------------------------------- ## +## Report this to info@codership.com ## +## --------------------------------- ## +_ASBOX + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + eval "$as_ac_Header=\$ac_header_preproc" +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } + +fi +if test `eval echo '${'$as_ac_Header'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + { echo "$as_me:$LINENO: checking for poll" >&5 +echo $ECHO_N "checking for poll... $ECHO_C" >&6; } +if test "${ac_cv_func_poll+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define poll to an innocuous variant, in case <limits.h> declares poll. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define poll innocuous_poll + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char poll (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef poll + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char poll (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined __stub_poll || defined __stub___poll +choke me +#endif + +int +main () +{ +return poll (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && + $as_test_x conftest$ac_exeext; then + ac_cv_func_poll=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_func_poll=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +fi +{ echo "$as_me:$LINENO: result: $ac_cv_func_poll" >&5 +echo "${ECHO_T}$ac_cv_func_poll" >&6; } +if test $ac_cv_func_poll = yes; then + POLL="POLL" +else + { { echo "$as_me:$LINENO: error: *** poll() not found! *** +See \`config.log' for more details." >&5 +echo "$as_me: error: *** poll() not found! *** +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + +fi + + +else + { { echo "$as_me:$LINENO: error: *** poll.h not found! *** +See \`config.log' for more details." >&5 +echo "$as_me: error: *** poll.h not found! *** +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + +fi + +done + + +if test "$enable_poll" == "no" +then + +for ac_header in sys/epoll.h +do +as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh` +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + { echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } +else + # Is the header compilable? +{ echo "$as_me:$LINENO: checking $ac_header usability" >&5 +echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6; } + +# Is the header present? +{ echo "$as_me:$LINENO: checking $ac_header presence" >&5 +echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <$ac_header> +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5 +echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: $ac_header: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## --------------------------------- ## +## Report this to info@codership.com ## +## --------------------------------- ## +_ASBOX + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + eval "$as_ac_Header=\$ac_header_preproc" +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } + +fi +if test `eval echo '${'$as_ac_Header'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + { echo "$as_me:$LINENO: checking for epoll_create" >&5 +echo $ECHO_N "checking for epoll_create... $ECHO_C" >&6; } +if test "${ac_cv_func_epoll_create+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define epoll_create to an innocuous variant, in case <limits.h> declares epoll_create. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define epoll_create innocuous_epoll_create + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char epoll_create (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef epoll_create + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char epoll_create (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined __stub_epoll_create || defined __stub___epoll_create +choke me +#endif + +int +main () +{ +return epoll_create (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && + $as_test_x conftest$ac_exeext; then + ac_cv_func_epoll_create=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_func_epoll_create=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +fi +{ echo "$as_me:$LINENO: result: $ac_cv_func_epoll_create" >&5 +echo "${ECHO_T}$ac_cv_func_epoll_create" >&6; } +if test $ac_cv_func_epoll_create = yes; then + POLL="EPOLL" +else + { echo "$as_me:$LINENO: WARNING: epoll API not found, trying poll()" >&5 +echo "$as_me: WARNING: epoll API not found, trying poll()" >&2;} + +fi + + + +fi + +done + +fi + if test "$enable_splice" == "yes" then @@ -6322,10 +6819,25 @@ # Many feature checks are broken and issue warnings. # If we want checks to pass we have to put this at the very end. -CFLAGS="$CFLAGS -Wall -Werror" +CFLAGS="$CFLAGS -Wall -Werror -DUSE_$POLL " -CXXFLAGS="$CFLAGS $CXXFLAGS" +#CXXFLAGS="$CFLAGS $CXXFLAGS" +#AC_SUBST(CXXFLAGS) +{ echo "$as_me:$LINENO: ----------------------" >&5 +echo "$as_me: ----------------------" >&6;} +{ echo "$as_me:$LINENO: poll method used: $POLL" >&5 +echo "$as_me: poll method used: $POLL" >&6;} +{ echo "$as_me:$LINENO: splice() enabled: $enable_splice" >&5 +echo "$as_me: splice() enabled: $enable_splice" >&6;} +{ echo "$as_me:$LINENO: stats enabled: $enable_stats" >&5 +echo "$as_me: stats enabled: $enable_stats" >&6;} +{ echo "$as_me:$LINENO: debug enabled: $enable_debug" >&5 +echo "$as_me: debug enabled: $enable_debug" >&6;} +{ echo "$as_me:$LINENO: CFLAGS = $CFLAGS" >&5 +echo "$as_me: CFLAGS = $CFLAGS" >&6;} +{ echo "$as_me:$LINENO: ----------------------" >&5 +echo "$as_me: ----------------------" >&6;} ac_config_files="$ac_config_files Makefile doc/Makefile src/Makefile" @@ -6425,17 +6937,17 @@ LTLIBOBJS=$ac_ltlibobjs -if test -z "${ENABLE_DEBUG_TRUE}" && test -z "${ENABLE_DEBUG_FALSE}"; then - { { echo "$as_me:$LINENO: error: conditional \"ENABLE_DEBUG\" was never defined. +if test -z "${ENABLE_SPLICE_TRUE}" && test -z "${ENABLE_SPLICE_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"ENABLE_SPLICE\" was never defined. Usually this means the macro was only invoked conditionally." >&5 -echo "$as_me: error: conditional \"ENABLE_DEBUG\" was never defined. +echo "$as_me: error: conditional \"ENABLE_SPLICE\" was never defined. Usually this means the macro was only invoked conditionally." >&2;} { (exit 1); exit 1; }; } fi -if test -z "${ENABLE_SPLICE_TRUE}" && test -z "${ENABLE_SPLICE_FALSE}"; then - { { echo "$as_me:$LINENO: error: conditional \"ENABLE_SPLICE\" was never defined. +if test -z "${ENABLE_DEBUG_TRUE}" && test -z "${ENABLE_DEBUG_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"ENABLE_DEBUG\" was never defined. Usually this means the macro was only invoked conditionally." >&5 -echo "$as_me: error: conditional \"ENABLE_SPLICE\" was never defined. +echo "$as_me: error: conditional \"ENABLE_DEBUG\" was never defined. Usually this means the macro was only invoked conditionally." >&2;} { (exit 1); exit 1; }; } fi @@ -6767,7 +7279,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by glb $as_me 0.6.0, which was +This file was extended by glb $as_me 0.7.2, which was generated by GNU Autoconf 2.61. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -6820,7 +7332,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -glb config.status 0.6.0 +glb config.status 0.7.2 configured by $0, generated by GNU Autoconf 2.61, with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" @@ -7072,10 +7584,10 @@ AMTAR!$AMTAR$ac_delim am__tar!$am__tar$ac_delim am__untar!$am__untar$ac_delim -ENABLE_DEBUG_TRUE!$ENABLE_DEBUG_TRUE$ac_delim -ENABLE_DEBUG_FALSE!$ENABLE_DEBUG_FALSE$ac_delim ENABLE_SPLICE_TRUE!$ENABLE_SPLICE_TRUE$ac_delim ENABLE_SPLICE_FALSE!$ENABLE_SPLICE_FALSE$ac_delim +ENABLE_DEBUG_TRUE!$ENABLE_DEBUG_TRUE$ac_delim +ENABLE_DEBUG_FALSE!$ENABLE_DEBUG_FALSE$ac_delim ENABLE_STATS_TRUE!$ENABLE_STATS_TRUE$ac_delim ENABLE_STATS_FALSE!$ENABLE_STATS_FALSE$ac_delim CXX!$CXX$ac_delim

@@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.50) -AC_INIT(glb, 0.6.0, info@codership.com) +AC_INIT(glb, 0.7.2, info@codership.com) AC_CONFIG_SRCDIR([src/glb_main.c]) AC_CONFIG_HEADER([config.h]) AC_CANONICAL_SYSTEM @@ -11,6 +11,26 @@ #AM_GNU_GETTEXT #AM_GNU_GETTEXT_VERSION(0.17) +# Check for poll method +AC_ARG_ENABLE([poll], + [AC_HELP_STRING([--enable-poll], + [use poll() for polling (default: use epoll() if available)] + ) + ], + [], + enable_poll="no") + +# Check for splice() +AC_ARG_ENABLE(splice, + AC_HELP_STRING([--enable-splice], + [use splice() [[default=disabled]]]),, + enable_splice="no") +if test "$enable_splice" == "yes" +then + CFLAGS="$CFLAGS -D_GNU_SOURCE -DGLB_USE_SPLICE" +fi +AM_CONDITIONAL(ENABLE_SPLICE, test "$enable_splice" != "no") + # Check for debug AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug], @@ -18,26 +38,15 @@ enable_debug="no") if test "$enable_debug" == "yes" then - CFLAGS="-g -O1 -fno-inline " + CFLAGS="-g -O0 -fno-inline " else CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} CFLAGS="$CFLAGS -DNDEBUG " fi AM_CONDITIONAL(ENABLE_DEBUG, test "$enable_debug" != "no") -# Check for splice() -AC_ARG_ENABLE(splice, - AC_HELP_STRING([--enable-splice], - [use splice() [[default=disabled]]]),, - enable_splice="no") -if test "$enable_splice" == "yes" -then - CFLAGS="$CFLAGS -D_GNU_SOURCE -DGLB_USE_SPLICE" -fi -AM_CONDITIONAL(ENABLE_SPLICE, test "$enable_splice" != "no") - # Check for stats -AC_ARG_ENABLE(splice, +AC_ARG_ENABLE(stats, AC_HELP_STRING([--enable-stats], [use extensive pool statistics [[default=disabled]]]),, enable_stats="no") @@ -53,7 +62,6 @@ AC_PROG_CC #AC_PROG_LIBTOOL -# FIXME: Replace `main' with a function in `-lpthread': AC_CHECK_LIB([pthread], [pthread_testcancel],, AC_MSG_ERROR([*** POSIX threads not found! ***])) @@ -73,7 +81,28 @@ AC_FUNC_MALLOC AC_HEADER_STDC AC_FUNC_REALLOC -AC_CHECK_FUNCS([gettimeofday memset strcasecmp strdup strncasecmp strtol]) +AC_CHECK_FUNCS([gettimeofday memset strdup strcasecmp strncasecmp strtol]) + +# Check for poll()/epoll() +AC_CHECK_HEADERS([poll.h], + [AC_CHECK_FUNC([poll],[POLL="POLL"], + [AC_MSG_FAILURE([*** poll() not found! ***])] + ) + ], + [AC_MSG_FAILURE([*** poll.h not found! ***])] + ) + +if test "$enable_poll" == "no" +then + AC_CHECK_HEADERS([sys/epoll.h], + [AC_CHECK_FUNC([epoll_create], + [POLL="EPOLL"], + [AC_MSG_WARN([epoll API not found, trying poll()])] + ) + ] + ) +fi + if test "$enable_splice" == "yes" then AC_CHECK_FUNCS([splice]) @@ -81,10 +110,18 @@ # Many feature checks are broken and issue warnings. # If we want checks to pass we have to put this at the very end. -CFLAGS="$CFLAGS -Wall -Werror" +CFLAGS="$CFLAGS -Wall -Werror -DUSE_$POLL " AC_SUBST(CFLAGS) -CXXFLAGS="$CFLAGS $CXXFLAGS" -AC_SUBST(CXXFLAGS) +#CXXFLAGS="$CFLAGS $CXXFLAGS" +#AC_SUBST(CXXFLAGS) + +AC_MSG_NOTICE([----------------------]) +AC_MSG_NOTICE([poll method used: $POLL]) +AC_MSG_NOTICE([splice() enabled: $enable_splice]) +AC_MSG_NOTICE([stats enabled: $enable_stats]) +AC_MSG_NOTICE([debug enabled: $enable_debug]) +AC_MSG_NOTICE([CFLAGS = $CFLAGS]) +AC_MSG_NOTICE([----------------------]) AC_CONFIG_FILES([Makefile doc/Makefile

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_cmd.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_cmd.c 63 2009-04-18 12:09:59Z alex $ */ #include <stddef.h> // ptrdiff_t @@ -51,9 +51,8 @@ }; // Some constants -static const int cmd_list_separator = ','; -static const long cmd_ip_len_max = 256; -static const long cmd_port_max = (1<<16) - 1; +#define cmd_ip_len_max 256 +#define cmd_port_max ((1<<16) - 1) void glb_cmd_help (FILE* out, const char* progname) @@ -62,32 +61,30 @@ "Usage:\n %s [OPTIONS] LISTEN_ADDRESS " "[DESTINATION_LIST]\nOPTIONS:\n", progname); fprintf (out, - " --help this help message\n"); + " --help this help message.\n"); fprintf (out, - " --fifo <fifo name> name of the FIFO file for control\n"); + " --fifo <fifo name> name of the FIFO file for control.\n"); fprintf (out, " --control [HOST:]PORT " - "listen for control requests on this address\n" - " " - "(default: 127.0.0.1:<LISTEN_PORT + 1>\n"); + "listen for control requests on this address.\n"); fprintf (out, " --threads N " - "number of working threads (number of CPU cores)\n"); + "number of working threads (connection pools).\n"); fprintf (out, " --source_tracking " "turn on source tracking: route connections from one\n" - " source to the same destination\n"); + " source to the same destination.\n"); fprintf (out, - " --verbose turn on verbose reporting\n"); + " --verbose turn on verbose reporting.\n"); fprintf (out, - " --version print program version\n"); + " --version print program version.\n"); fprintf (out, "LISTEN_ADDRESS:\n" " [IP:]PORT " - "where to listen for incoming TCP connections\n"); + "where to listen for incoming TCP connections.\n"); fprintf (out, "DESTINATION_LIST:\n" " [H1[:P1[:W1]]] [H2[:P2[:W2]]]... " " - a space-separated list of destinations\n" - " in the form address:port:weight\n"); + " in the form address:port:weight.\n"); exit (EXIT_FAILURE); } @@ -99,7 +96,7 @@ fprintf (out, "Incoming address: %s, ", glb_socket_addr_to_string (&cmd->inc_addr)); fprintf (out, "control FIFO: %s\n", cmd->fifo_name); - fprintf (out, "Control address: %s\n", + fprintf (out, "Control address: %s\n", cmd->ctrl_set ? glb_socket_addr_to_string (&cmd->ctrl_addr) : "none"); fprintf (out, "Number of threads: %lu, source tracking: %s, verbose: %s, " @@ -157,7 +154,7 @@ return glb_socket_addr_init (addr, addr_str, port); } -// parses comma separated list of destinations, reallocates and fills conf +// parses array list of destinations static glb_cmd_t* cmd_parse_dst_list (const char* dst_list[], size_t n_dst, @@ -165,25 +162,12 @@ { glb_cmd_t* ret = NULL; size_t i; - const size_t max_dst_len = 256; // addr:port:weight\0 - char dst_str[max_dst_len + 1] = { 0, }; - ptrdiff_t dst_len; ret = calloc (sizeof(*ret) + n_dst * sizeof(glb_dst_t), 1); if (ret) { for (i = 0; i < n_dst; i++) { - - dst_len = strlen (dst_list[i]); - - if (dst_len > max_dst_len) { - fprintf (stderr, "Destination spec too long: %s\n",dst_list[i]); - free (ret); - return NULL; - } - - strncpy (dst_str, dst_list[i], dst_len); - - switch (glb_dst_parse (&ret->dst[i], dst_str, default_port)) { + + switch (glb_dst_parse (&ret->dst[i], dst_list[i], default_port)) { case 1: // default port is assigned glb_dst_parse() case 2: @@ -191,7 +175,7 @@ case 3: break; default: // error parsing destination - fprintf (stderr, "Invalid destination spec: %s\n", dst_str); + fprintf (stderr, "Invalid destination spec: %s\n", dst_list[i]); free (ret); return NULL; } @@ -282,6 +266,17 @@ } inc_port = glb_socket_addr_get_port (&tmp.inc_addr); +#if 0 // don't open socket by default for security considerations. + // if control address was not specified + if (!tmp.ctrl_set) { + char port[6] = { 0, }; + snprintf (port, 5, "%hu", inc_port + 1); + if (cmd_parse_addr (&tmp.ctrl_addr, port, cmd_ctrl_addr_default)) + return NULL; + tmp.ctrl_set = true; + } +#endif + // if number of threads was not specified if (!tmp.n_threads) tmp.n_threads = 1;

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_control.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_control.c 65 2009-06-06 21:16:51Z alex $ */ // keep asserts here for now @@ -16,20 +16,31 @@ #include <errno.h> #include <stdlib.h> #include <string.h> -#include <sys/socket.h> -#include <sys/un.h> #include <sys/stat.h> #include <fcntl.h> #include <ctype.h> +// unfotunately I see no way to use glb_pool.c polling code in here +// so it is yet another implementation +#include <poll.h> +typedef struct pollfd pollfd_t; + #include "glb_log.h" #include "glb_signal.h" #include "glb_control.h" extern bool glb_verbose; +static const char ctrl_getinfo_cmd[] = "getinfo"; static const char ctrl_getstat_cmd[] = "getstat"; +typedef enum ctrl_fd +{ + CTRL_FIFO = 0, + CTRL_LISTEN, + CTRL_MAX = 32 // max 30 simultaneous control connections +} ctrl_fd_t; + struct glb_ctrl { pthread_t thread; @@ -39,28 +50,51 @@ glb_router_t* router; glb_pool_t* pool; int fd_max; - fd_set fds; + pollfd_t fds[CTRL_MAX]; uint16_t default_port; }; static void ctrl_add_client (glb_ctrl_t* ctrl, int fd) { - assert (ctrl->fd_max != fd); - FD_SET (fd, &ctrl->fds); - if (fd > ctrl->fd_max) ctrl->fd_max = fd; + assert (ctrl->fd_max < CTRL_MAX); + + ctrl->fds[ctrl->fd_max].fd = fd; + ctrl->fds[ctrl->fd_max].events = POLLIN; + ctrl->fds[ctrl->fd_max].revents = 0; + + ctrl->fd_max++; + + if (CTRL_MAX == ctrl->fd_max) // no more clients + ctrl->fds[CTRL_LISTEN].events = 0; } static void ctrl_del_client (glb_ctrl_t* ctrl, int fd) { - assert (ctrl->fd_max >= fd); - FD_CLR (fd, &ctrl->fds); - if (fd == ctrl->fd_max) { // find next highest fd - while (!FD_ISSET (ctrl->fd_max, &ctrl->fds)) ctrl->fd_max--; + int idx; + + assert (CTRL_MAX >= ctrl->fd_max); + assert (ctrl->fd_max > CTRL_LISTEN); + + for (idx = 1; idx < ctrl->fd_max; idx++) { + if (fd == ctrl->fds[idx].fd) { + close (ctrl->fds[idx].fd); + ctrl->fd_max--; + ctrl->fds[idx] = ctrl->fds[ctrl->fd_max]; + ctrl->fds[ctrl->fd_max].fd = 0; + ctrl->fds[ctrl->fd_max].events = 0; + ctrl->fds[ctrl->fd_max].revents = 0; + break; + } } - assert (ctrl->fd_max < fd); - close (fd); + + if (!(ctrl->fd_max < CTRL_MAX)) { + glb_log_fatal ("Failed to cleanup control connection."); + abort(); + }; + + ctrl->fds[CTRL_LISTEN].events = POLLIN; } static inline void @@ -104,8 +138,13 @@ } } - if (!strncasecmp (ctrl_getstat_cmd, req, strlen(ctrl_getstat_cmd))) { - glb_router_print_stats (ctrl->router, req, sizeof(req)); + if (!strncasecmp (ctrl_getinfo_cmd, req, strlen(ctrl_getinfo_cmd))) { + glb_router_print_info (ctrl->router, req, sizeof(req)); + ctrl_respond (ctrl, fd, req); + return 0; + } + else if (!strncasecmp (ctrl_getstat_cmd, req, strlen(ctrl_getstat_cmd))) { + glb_pool_print_stats (ctrl->pool, req, sizeof(req)); ctrl_respond (ctrl, fd, req); return 0; } @@ -126,38 +165,50 @@ ctrl_respond (ctrl, fd, "Error\n"); return 0; } - ctrl_respond (ctrl, fd, "OK\n"); - if (dst.weight < 0) { + ctrl_respond (ctrl, fd, "Ok\n"); + + if (dst.weight < 0.0) { // destination was removed from router, drop all connections to it glb_pool_drop_dst (ctrl->pool, &dst.addr); } + return 0; } } +// returns true if fd is ready to read +static inline bool +ctrl_fd_isset (glb_ctrl_t* ctrl, int idx) +{ + return (ctrl->fds[idx].revents & POLLIN); +} + static void* ctrl_thread (void* arg) { glb_ctrl_t* ctrl = arg; while (!glb_terminate) { - long ret; - int client_sock; - struct sockaddr client; - socklen_t client_size; - fd_set fds = ctrl->fds; - struct timeval timeout = { 1, 0 }; + long ret; - ret = select (ctrl->fd_max + 1, &fds, NULL, NULL, &timeout); + // Timeout is needed to gracefully shut down + ret = poll (ctrl->fds, ctrl->fd_max, 1000 /* ms */); if (ret < 0) { - perror ("error waiting for connections"); + glb_log_error ("Error waiting for connections: %d (%s)", + errno, strerror(errno)); goto err; //? } else if (0 == ret) continue; - if (ctrl->inet_sock > 0 && FD_ISSET (ctrl->inet_sock, &fds)) { + if (ctrl->inet_sock > 0 && (ctrl_fd_isset (ctrl, CTRL_LISTEN))) { + // new network client + int client_sock; + struct sockaddr client; + socklen_t client_size; + client_sock = accept (ctrl->inet_sock, &client, &client_size); + if (client_sock < 0) { glb_log_error ("Ctrl: failed to accept connection: %d (%s)", errno, strerror (errno)); @@ -175,10 +226,12 @@ } else { int fd; - for (fd = 1; fd <= ctrl->fd_max; fd++) { // find fd - if (FD_ISSET (fd, &fds)) break; + for (fd = CTRL_FIFO; fd <= ctrl->fd_max; fd++) { // find fd + if (ctrl_fd_isset (ctrl, fd)) { + assert (fd != CTRL_LISTEN); + if (ctrl_handle_request (ctrl, ctrl->fds[fd].fd)) goto err; + } } - if (ctrl_handle_request (ctrl, fd)) goto err; continue; } @@ -247,21 +300,17 @@ ret->fifo_name = fifo_name; ret->fifo = fifo; ret->inet_sock = inet_sock; - ret->fd_max = fifo > inet_sock ? fifo : inet_sock; + ret->fd_max = fifo > inet_sock ? 1 : 2; ret->default_port = port; - FD_ZERO (&ret->fds); - FD_SET (ret->fifo, &ret->fds); - if (ret->inet_sock > 0) FD_SET (ret->inet_sock, &ret->fds); - -#if 0 // remove - for (fd = 0; fd < FD_SETSIZE; fd++) { - if (FD_ISSET (fd, &ret->fds)) - printf ("Ctrl: set fd %d\n", fd); - } - printf ("Ctrl: fd_max = %d\n", ret->fd_max); - printf ("Ctrl: ctrl object: %p\n", ret); -#endif + ret->fds[CTRL_FIFO].fd = ret->fifo; + ret->fds[CTRL_FIFO].events = POLLIN; + + if (ret->inet_sock > 0) { + ret->fds[CTRL_LISTEN].fd = ret->inet_sock; + ret->fds[CTRL_LISTEN].events = POLLIN; + } + if (pthread_create (&ret->thread, NULL, ctrl_thread, ret)) { glb_log_error ("Failed to launch ctrl thread."); free (ret);

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_dst.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_dst.c 59 2009-04-16 10:30:17Z alex $ */ #include <stddef.h> // ptrdiff_t @@ -12,11 +12,11 @@ #include "glb_dst.h" // Some constants -static const int dst_separator = ':'; -static const ptrdiff_t dst_ip_len_max = 256; -static const ptrdiff_t dst_ip_len_min = 1; -static const ulong dst_port_max = (1 << 16) - 1; -static const long dst_default_weight = 1; +#define dst_separator ':' +#define dst_ip_len_max 256 +#define dst_ip_len_min 1 +#define dst_port_max ((1 << 16) - 1) +#define dst_default_weight 1.0 // parses addr:port:weight string, stores in dst // returns number of parsed fields or negative error code @@ -72,7 +72,7 @@ // parse weight assert (*endptr == dst_separator); token = endptr + 1; - dst->weight = strtoul (token, &endptr, 10); + dst->weight = strtod (token, &endptr); if (*endptr != '\0') { perror ("Weight field doesn't consist only of numbers"); return -EINVAL;

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_dst.h 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_dst.h 59 2009-04-16 10:30:17Z alex $ */ #ifndef _glb_dst_h_ @@ -16,9 +16,9 @@ typedef struct glb_dst { glb_sockaddr_t addr; // destination address in prepared form - long weight; // >0: connection allocation weight (def: 1) + double weight; // >0: connection allocation weight (def: 1) // 0: no new conns, but keep existing (drain) - // -1: discard destination entirely + // <0: discard destination entirely } glb_dst_t; /*! @@ -35,7 +35,7 @@ } static inline void -glb_dst_set_weight (glb_dst_t* dst, long weight) +glb_dst_set_weight (glb_dst_t* dst, double weight) { dst->weight = weight; } @@ -49,7 +49,7 @@ static inline void glb_dst_print (char* buf, size_t buf_len, const glb_dst_t* dst) { - snprintf (buf, buf_len, "%s, w: %ld", + snprintf (buf, buf_len, "%s, w: %5.3f", glb_socket_addr_to_string(&dst->addr), dst->weight); buf[buf_len - 1] = '\0'; }

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_listener.c 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_listener.c 65 2009-06-06 21:16:51Z alex $ */ #include <pthread.h> @@ -9,6 +9,8 @@ #include <unistd.h> #include <errno.h> #include <string.h> +#include <poll.h> +typedef struct pollfd pollfd_t; #include "glb_log.h" #include "glb_listener.h" @@ -27,11 +29,7 @@ listener_thread (void* arg) { glb_listener_t* listener = arg; - fd_set fds; - - // shoudl be no need to refresh the fd set - FD_ZERO (&fds); - FD_SET (listener->sock, &fds); + pollfd_t pollfd = { .fd = listener->sock, .events = POLLIN, .revents = 0 }; while (1) { long ret; @@ -41,28 +39,27 @@ int server_sock; glb_sockaddr_t server; - ret = select (listener->sock + 1, &fds, NULL, NULL, NULL); + ret = poll (&pollfd, 1, -1); if (ret < 0) { - glb_log_error ("error waiting for connections: %d (%s)", + glb_log_error ("Error waiting for connections: %d (%s)", errno, strerror (errno)); - FD_SET (listener->sock, &fds); goto err; //? } assert (1 == ret); - assert (FD_ISSET (listener->sock, &fds)); + assert (pollfd.revents & POLLIN); client_sock = accept (listener->sock, (struct sockaddr*) &client, &client_size); if (client_sock < 0) { - glb_log_error ("Listener: failed to accept connection: %d (%s)", + glb_log_error ("Failed to accept connection: %d (%s)", errno, strerror (errno)); goto err; } server_sock = glb_router_connect (listener->router, &server); if (server_sock < 0) { - glb_log_error("Listener: failed to connect to destination: %d (%s)", + glb_log_error("Failed to connect to destination: %d (%s)", errno, strerror(errno)); goto err1; } @@ -70,16 +67,16 @@ ret = glb_pool_add_conn (listener->pool, client_sock, server_sock, &server); if (ret < 0) { - glb_log_error ("Listener: failed to add connection to pool: " + glb_log_error ("Failed to add connection to pool: " "%d (%s)", errno, strerror (errno)); goto err2; } if (glb_verbose) { - fprintf (stderr, "Listener: accepted connection from %s ", - glb_socket_addr_to_string (&client)); - fprintf (stderr, "to %s\n", - glb_socket_addr_to_string (&server)); + glb_log_info ("Accepted connection from %s ", + glb_socket_addr_to_string (&client)); + glb_log_info ("to %s\n", + glb_socket_addr_to_string (&server)); } continue;

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_log.h 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_log.h 57 2009-04-15 17:36:21Z alex $ */ #ifndef _glb_log_h_ @@ -51,9 +51,12 @@ glb_log (GLB_LOG_INFO, __FILE__, __PRETTY_FUNCTION__, __LINE__,\ format, ## __VA_ARGS__, NULL) +#ifdef NDEBUG +#define glb_log_debug(format, ...) +#else // DEBUG #define glb_log_debug(format, ...) \ glb_log (GLB_LOG_DEBUG, __FILE__, __PRETTY_FUNCTION__, __LINE__,\ format, ## __VA_ARGS__, NULL) - +#endif // DEBUG #endif // _glb_log_h_

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_main.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_main.c 59 2009-04-16 10:30:17Z alex $ */ #include <unistd.h> // for sleep() @@ -85,10 +85,10 @@ if (!cmd->daemonize) { char stats[BUFSIZ]; - glb_router_print_stats (router, stats, BUFSIZ); + glb_router_print_info (router, stats, BUFSIZ); puts (stats); - glb_pool_print_stats (pool, stats, BUFSIZ); + glb_pool_print_info (pool, stats, BUFSIZ); puts (stats); }

@@ -1,11 +1,9 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_pool.c 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_pool.c 61 2009-04-17 22:23:24Z alex $ */ -#include <stdlib.h> -#include <sys/select.h> // for select() and FD_SET #include <pthread.h> #include <stdio.h> #include <string.h> @@ -13,12 +11,25 @@ #include <unistd.h> #include <errno.h> #include <assert.h> -#include <sys/time.h> + +#ifndef USE_EPOLL + #ifndef USE_POLL + #error "Neither USE_POLL nor USE_EPOLL defined!" + #else + #include <poll.h> + typedef struct pollfd pollfd_t; + #endif +#else + #include <sys/epoll.h> + typedef struct epoll_event pollfd_t; +#endif #ifdef GLB_USE_SPLICE #include <fcntl.h> #endif +#include "glb_time.h" +#include "glb_log.h" #include "glb_pool.h" extern bool glb_verbose; @@ -28,6 +39,7 @@ POOL_CTL_ADD_CONN, POOL_CTL_DROP_DST, POOL_CTL_SHUTDOWN, + POOL_CTL_STATS, POOL_CTL_MAX } pool_ctl_code_t; @@ -37,25 +49,12 @@ void* data; } pool_ctl_t; -typedef struct pool_stats -{ - size_t recv_bytes; - size_t n_recv; - size_t send_bytes; - size_t n_send; - size_t sel_reads; - size_t sel_writes; - size_t n_select; -} pool_stats_t; - -#ifdef GLB_POOL_STATS -static pool_stats_t zero_stats = { 0, }; -#endif - typedef struct pool_conn_end { bool inc; // to differentiate between the ends int sock; // fd of connection + int fds_idx; // index in the file descriptor set (for poll()) + uint32_t events; // events waited by descriptor size_t sent; size_t total; glb_sockaddr_t dst_addr; // destinaiton id @@ -65,79 +64,208 @@ uint8_t buf[]; // has pool_buf_size } pool_conn_end_t; +#define POOL_MAX_FD (1 << 16) // highest possible file descriptor + 1 + // only affects the map size + // We want to allocate memory for both ends in one malloc() call and have it // nicely aligned. This is presumably a page multiple and // should be enough for two ethernet frames (what about jumbo?) -#define pool_conn_size (BUFSIZ << 2) -const size_t pool_buf_size = (pool_conn_size/2 - sizeof(pool_conn_end_t)); +#define pool_end_size (BUFSIZ) +#define pool_conn_size (pool_end_size << 2) +#define pool_buf_size (pool_end_size - sizeof(pool_conn_end_t)) typedef struct pool { - long id; - pthread_t thread; - pthread_mutex_t lock; - pthread_cond_t cond; - int ctl_recv; // receive commands - pool thread - int ctl_send; // send commands to pool - other function - volatile ulong n_conns; // how many connecitons this pool serves - fd_set fds_read; - fd_set fds_write; - int fd_max; - int fd_min; - glb_router_t* router; -#ifdef GLB_POOL_STATS - volatile pool_stats_t stats; -#endif - pool_conn_end_t* route_map[FD_SETSIZE]; // looking for connection ctx by fd + long id; + pthread_t thread; + pthread_mutex_t lock; + pthread_cond_t cond; + int ctl_recv; // fd to receive commands in pool thread + int ctl_send; // fd to send commands to pool - other function + volatile ulong n_conns; // how many connecitons this pool serves +#ifdef USE_EPOLL + int epoll_fd; +#endif + pollfd_t* pollfds; + size_t pollfds_len; + int fd_max; + glb_router_t* router; + glb_pool_stats_t stats; + + pool_conn_end_t* route_map[ POOL_MAX_FD ]; // connection ctx look-up by fd } pool_t; struct glb_pool { pthread_mutex_t lock; ulong n_pools; - struct timeval begin; + glb_time_t last_info; + glb_time_t last_stats; pool_t pool[]; // pool array, can't be changed in runtime }; +typedef enum pool_fd_ops +{ +#ifdef USE_EPOLL + POOL_FD_READ = EPOLLIN, + POOL_FD_WRITE = EPOLLOUT, +#else // POLL + POOL_FD_READ = POLLIN, + POOL_FD_WRITE = POLLOUT, +#endif // POLL + POOL_FD_RW = POOL_FD_READ | POOL_FD_WRITE +} pool_fd_ops_t; + +//#define FD_SETSIZE 1024; // leater get it from select.h + +static const pollfd_t zero_pollfd = { 0, }; + +/*! + * @return negative error code or the index of file descriptor in the set + */ +static inline long +pool_fds_add (pool_t* pool, int fd, pool_fd_ops_t events) +{ + int ret; + + assert (fd > 0); + assert (pool->fd_max <= pool->pollfds_len); + + if (pool->fd_max == pool->pollfds_len) { // allocate more memory + void* tmp; + size_t tmp_len = pool->pollfds_len + FD_SETSIZE; + + tmp = realloc (pool->pollfds, tmp_len * sizeof(pollfd_t)); + if (NULL == tmp) { + glb_log_fatal ("Failed to (re)allocate %d pollfds: out of memory", + tmp_len); + return -ENOMEM; + } + + memset (((pollfd_t*)tmp) + pool->fd_max, 0, + (tmp_len - pool->fd_max) * sizeof(pollfd_t)); + + pool->pollfds = tmp; + pool->pollfds_len = tmp_len; + } + +#ifdef USE_EPOLL + struct epoll_event add_event = { .events = events, { .fd = fd }}; + + ret = epoll_ctl (pool->epoll_fd, EPOLL_CTL_ADD, fd, &add_event); + if (ret) { + glb_log_error ("epoll_ctl (%d, EPOLL_CTL_ADD, %d, {%d, %llu}) failed: " + "%d (%s)", + pool->epoll_fd, fd, add_event.events, add_event.data.u64, + errno, strerror (errno)); + return -errno; + } +#else // POLL + pool->pollfds[pool->fd_max].fd = fd; + pool->pollfds[pool->fd_max].events = events; +#endif // POLL + + ret = pool->fd_max; + + pool->fd_max++; // track how many descriptors are there + + return ret; +} + +// returns corresponding pool_conn_end_t* +static inline pool_conn_end_t* +pool_conn_end_by_fd (pool_t* pool, int fd) +{ + // map points to the other end, but that's enough + register pool_conn_end_t* other_end = pool->route_map[fd]; + if (other_end->inc) { + return (pool_conn_end_t*)((uint8_t*)other_end + pool_end_size); + } else { + return (pool_conn_end_t*)((uint8_t*)other_end - pool_end_size); + } +} + +// remove file descriptor from file descriptor set +static inline long +pool_fds_del (pool_t* pool, pool_conn_end_t* end) +{ + pool->fd_max--; // pool->fd_max is now the index of the last pollfd + +#ifdef USE_EPOLL + long ret = epoll_ctl (pool->epoll_fd, EPOLL_CTL_DEL, end->sock, NULL); + if (ret) { + glb_log_error ("epoll_ctl (%d, EPOLL_CTL_DEL, %d, NULL) failed: " + "%d (%s)", + pool->epoll_fd, end->sock, errno, strerror (errno)); + return -errno; + } +#else // POLL + assert (end->fds_idx <= pool->fd_max); + + /* + * pay attention here: the last pollfd that we're moving may have not been + * checked yet + */ + + // copy the last pollfd in place of the deleted + pool->pollfds[end->fds_idx] = pool->pollfds[pool->fd_max]; + // from this pollfd find its fd and from route_map by fd find its + // pool_conn_end struct and in that struct update fds_idx to point at + // a new position. + pool_conn_end_by_fd(pool, pool->pollfds[end->fds_idx].fd)->fds_idx = + end->fds_idx; + // zero-up the last pollfd + pool->pollfds[pool->fd_max] = zero_pollfd; +#endif // POLL + + return 0; +} + +static inline void +pool_fds_set_events (pool_t* pool, pool_conn_end_t* end) +{ +#ifdef USE_EPOLL + struct epoll_event event = { .events = end->events, { .fd = end->sock } }; + if (epoll_ctl (pool->epoll_fd, EPOLL_CTL_MOD, end->sock, &event)) { + glb_log_fatal ("epoll_ctl(%d, EPOLL_CTL_MOD, %d, {%d, %llu}) failed: " + "%d (%s)", + pool->epoll_fd, end->sock, event.events, event.data.u64, + errno, strerror(errno)); + abort(); + } +#else // POLL + pool->pollfds[end->fds_idx].events = end->events; +#endif // POLL +} + +static inline long +pool_fds_wait (pool_t* pool) +{ +#ifdef USE_EPOLL + return epoll_wait (pool->epoll_fd, pool->pollfds, pool->fd_max, -1); +#else // POLL + return poll (pool->pollfds, pool->fd_max, -1); +#endif // POLL +} + // performs necessary magic (adds end-to-end mapping, alters fd_max and fd_min) // when new file descriptor is added to fd_set static inline void pool_set_conn_end (pool_t* pool, pool_conn_end_t* end1, pool_conn_end_t* end2) { - assert (end1->sock < FD_SETSIZE); - assert (NULL == pool->route_map[end1->sock]); + end1->fds_idx = pool_fds_add (pool, end1->sock, POOL_FD_READ); + if (end1->fds_idx < 0) abort(); + end1->events = POOL_FD_READ; pool->route_map[end1->sock] = end2; - if (end1->sock > pool->fd_max) pool->fd_max = end1->sock; - if (end1->sock < pool->fd_min) pool->fd_min = end1->sock; - FD_SET (end1->sock, &pool->fds_read); } // removing traces of connection end - reverse to what pool_set_conn_end() did static inline void -pool_reset_conn_end (pool_t* pool, int fd) +pool_reset_conn_end (pool_t* pool, pool_conn_end_t* end) { - int i; - - FD_CLR (fd, &pool->fds_read); - FD_CLR (fd, &pool->fds_write); - pool->route_map[fd] = NULL; - - if (fd == pool->fd_max) { - // fd_max can't be less than pool->ctl_recv, because of select() - for (i = fd - 1; i > pool->ctl_recv; i--) { - if (pool->route_map[i] != NULL) break; - } - pool->fd_max = i; - } - - if (fd == pool->fd_min) { - for (i = fd + 1; i < pool->fd_max; i++) { - if (pool->route_map[i] != NULL) break; - } - pool->fd_min = i; - } - - close (fd); + pool_fds_del (pool, end); + close (end->sock); + pool->route_map[end->sock] = NULL; } static void @@ -147,13 +275,14 @@ int dst_fd = dst->sock; pool_conn_end_t* src = pool->route_map[dst_fd]; - pool->n_conns--; - - if (glb_verbose) { - fprintf (stderr, "Pool %ld: disconnecting from %s " - "(total pool connections: %ld)\n", pool->id, - glb_socket_addr_to_string (&dst->dst_addr), pool->n_conns); +#ifndef NDEBUG + if (notify_router && !src->inc) { + glb_log_warn ("Connection close from server"); } +#endif + + pool->n_conns--; + pool->stats.conns_closed++; if (notify_router) glb_router_disconnect (pool->router, &dst->dst_addr); @@ -163,32 +292,34 @@ close (src->splice[0]); close (src->splice[1]); #endif + // in reverse order to pool_set_conn_end() in pool_handle_add_conn() + pool_reset_conn_end (pool, dst); + pool_reset_conn_end (pool, src); + if (dst->inc) { free (dst); // frees both ends } else { assert (src->inc); - free (src); + free (src); // frees both ends } - - pool_reset_conn_end (pool, src_fd); - pool_reset_conn_end (pool, dst_fd); } static void pool_handle_add_conn (pool_t* pool, pool_ctl_t* ctl) { pool_conn_end_t* inc_end = ctl->data; - pool_conn_end_t* dst_end = ctl->data + pool_conn_size/2; + pool_conn_end_t* dst_end = ctl->data + pool_end_size; pool_set_conn_end (pool, inc_end, dst_end); pool_set_conn_end (pool, dst_end, inc_end); pool->n_conns++; // increment connection count + pool->stats.conns_opened++; + if (glb_verbose) { - fprintf (stderr, - "Pool %ld: added connection, (total pool connections: %ld)\n", - pool->id, pool->n_conns); + glb_log_info ("Pool %ld: added connection, " + "(total pool connections: %ld)", pool->id, pool->n_conns); } } @@ -197,44 +328,96 @@ { const glb_sockaddr_t* dst = ctl->data; int fd; + int count = pool->fd_max - 1; // ctl_recv is not in route_map - for (fd = pool->fd_min; fd <= pool->fd_max; fd++) { + for (fd = 0; count; fd++) { pool_conn_end_t* end = pool->route_map[fd]; - if (end && glb_socket_addr_is_equal(&end->dst_addr, dst)) { - // remove conn, but don't try to notify router 'cause it's already - // dropped this destination - pool_remove_conn (pool, fd, false); + assert (fd < POOL_MAX_FD); + + if (end) { + count--; + if (glb_socket_addr_is_equal(&end->dst_addr, dst)) { + // remove conn, but don't try to notify router 'cause it's + // already dropped this destination + pool_remove_conn (pool, fd, false); + count--; // 1 connection means 2 file descriptors + } } } } +static inline void +pool_handle_stats (pool_t* pool, pool_ctl_t* ctl) +{ + glb_pool_stats_t* stats = ctl->data; + pool->stats.n_conns = pool->n_conns; + glb_pool_stats_add (stats, &pool->stats); + pool->stats = glb_zero_stats; +} + +#define GLB_MUTEX_LOCK(mtx) \ +{ \ + int ret; \ + if ((ret = pthread_mutex_lock (mtx))) { \ + glb_log_fatal ("Failed to lock mutex: %d (%s)", ret, strerror(ret));\ + abort(); \ + } \ +} + +#define GLB_MUTEX_UNLOCK(mtx) \ +{ \ + int ret; \ + if ((ret = pthread_mutex_unlock (mtx))) { \ + glb_log_fatal ("Failed to unlock mutex: %d (%s)", ret, strerror(ret));\ + abort(); \ + } \ +} + static long -pool_handle_ctl (pool_t* pool, pool_ctl_t* ctl) +pool_handle_ctl (pool_t* pool) { - switch (ctl->code) { + pool_ctl_t ctl; + register long ret; + + // remove ctls from poll count to get only traffic polls + pool->stats.n_polls--; + + ret = read (pool->ctl_recv, &ctl, sizeof(ctl)); + + if (sizeof(ctl) != ret) { // incomplete ctl read, should never happen + glb_log_fatal ("Incomplete read from ctl, errno: %d (%s)", + errno, strerror (errno)); + abort(); + } + + switch (ctl.code) { case POOL_CTL_ADD_CONN: - pool_handle_add_conn (pool, ctl); + pool_handle_add_conn (pool, &ctl); break; case POOL_CTL_DROP_DST: - pool_handle_drop_dst (pool, ctl); + pool_handle_drop_dst (pool, &ctl); + break; + case POOL_CTL_STATS: + pool_handle_stats (pool, &ctl); break; default: // nothing else is implemented - fprintf (stderr, "Unsupported CTL: %d\n", ctl->code); + glb_log_warn ("Unsupported CTL: %d\n", ctl.code); } // Notify ctl sender - pthread_mutex_lock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); pthread_cond_signal (&pool->cond); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return 0; } static inline ssize_t -pool_send_data (pool_t* pool, pool_conn_end_t* dst, bool reset_fds_read) +pool_send_data (pool_t* pool, pool_conn_end_t* dst, pool_conn_end_t* src) { - ssize_t ret; + ssize_t ret; + uint32_t dst_events = dst->events; #ifndef GLB_USE_SPLICE ret = send (dst->sock, &dst->buf[dst->sent], dst->total - dst->sent, @@ -245,46 +428,62 @@ #endif if (ret > 0) { -#ifdef GLB_POOL_STATS pool->stats.send_bytes += ret; -#endif + pool->stats.tx_bytes += dst->inc * ret; + dst->sent += ret; - if (dst->sent == dst->total) { // all data sent, reset pointers - dst->sent = dst->total = 0; - FD_CLR (dst->sock, &pool->fds_write); - } - else { -// perror ("Pool: incomplete send"); - FD_SET (dst->sock, &pool->fds_write); - } - if (reset_fds_read && (dst->total < pool_buf_size)) { - // some space exists in the buffer, - // reestablish src_fd in pool->fds_read - int src_sock = pool->route_map[dst->sock]->sock; - FD_SET (src_sock, &pool->fds_read); + if (dst->sent == dst->total) { // all data sent, reset pointers + dst->sent = dst->total = 0; + dst_events &= ~POOL_FD_WRITE; // clear WRITE flag + } + else { // there is unsent data left + glb_log_debug ("Setting WRITE flag on %s: sent = %zu, total = " + "%zu, bufsiz = %zu", dst->inc ? "client" : "server", + dst->sent, dst->total,pool_buf_size); + dst_events |= POOL_FD_WRITE; // set WRITE flag + } + + if (src && !(src->events & POOL_FD_READ) && + (dst->total < pool_buf_size)) { + // some space exists in the buffer, reestablish READ flag in src + src->events |= POOL_FD_READ; + pool_fds_set_events (pool, src); } } else { - switch (errno) { + ret = -errno; + switch (-ret) { case ESPIPE: case EBUSY: case EINTR: case ENOBUFS: case EAGAIN: - FD_SET (dst->sock, &pool->fds_write); + glb_log_debug ("Send data error: %d (%s)", -ret, strerror(-ret)); + dst_events |= POOL_FD_WRITE; ret = 0; // pretend nothing happened break; case EPIPE: + glb_log_debug ("pool_remove_conn() from pool_send_data()"); pool_remove_conn(pool, dst->sock, true); break; default: - perror ("Pool: send failed"); - printf ("ALARM!!! missed case: errno = %d\n", errno); + glb_log_warn ("Send data failed, unhandled error: %d (%s)", + -ret, strerror(-ret)); } } -#ifdef GLB_POOL_STATS - pool->stats.n_send++; -#endif + + pool->stats.n_send++; + + if (dst_events != dst->events) { // events changed + glb_log_debug ("Old flags on %s: %s %s", dst->inc ? "client":"server", + dst->events & POOL_FD_READ ? "POOL_FD_READ":"", + dst->events & POOL_FD_WRITE ? "POOL_FD_WRITE":""); + dst->events = dst_events; + pool_fds_set_events (pool, dst); + glb_log_debug ("New flags on %s: %s %s", dst->inc ? "client":"server", + dst->events & POOL_FD_READ ? "POOL_FD_READ":"", + dst->events & POOL_FD_WRITE ? "POOL_FD_WRITE":""); + } return ret; } @@ -296,6 +495,8 @@ ssize_t ret = 0; pool_conn_end_t* dst = pool->route_map[src_fd]; +// glb_log_debug ("pool_handle_read()"); + // first, try read data from source, if there's enough space if (dst->total < pool_buf_size) { #ifndef GLB_USE_SPLICE @@ -308,33 +509,44 @@ #endif if (ret > 0) { dst->total += ret; - // now try to send whatever we have - // (since we're here, we're in fds_read, no need to reset) - if (pool_send_data (pool, dst, false) < 0) { - // probably don't care what error is - perror ("pool_handle_read(): sending data"); + // now try to send whatever we have received so far + // (since we're here, POOL_FD_READ on src is not cleared, no need + // to set it once again) + ssize_t send_err; + if ((send_err = pool_send_data (pool, dst, NULL)) < 0) { + glb_log_warn ("pool_send_data(): %zd (%s)", + -send_err, strerror(-send_err)); } + if (dst->total == pool_buf_size) { - // no space for next read, remove from fds_read - FD_CLR (src_fd, &pool->fds_read); + // no space for next read, clear POOL_FD_READ + pool_conn_end_t* src = pool->route_map[dst->sock]; + assert (src->events & POOL_FD_READ); + src->events &= ~POOL_FD_READ; + pool_fds_set_events (pool, src); } -#ifdef GLB_POOL_STATS + pool->stats.recv_bytes += ret; -#endif + + // increment only if it's coming from incoming interface + pool->stats.rx_bytes += (!dst->inc) * ret; } else { if (0 == ret) { // socket closed, must close another end and cleanup +// glb_log_debug ("pool_remove_conn() from pool_handle_read()"); pool_remove_conn (pool, src_fd, true); - ret = -1; + ret = -EPIPE; } else { // some other error - if (errno != EAGAIN) - perror ("pool_handle_read(): receiving data"); + if (errno != EAGAIN) { + ret = -errno; + glb_log_warn ("pool_handle_read(): %zd (%s)", + -ret, strerror(-ret)); + } } } -#ifdef GLB_POOL_STATS + pool->stats.n_recv++; -#endif } return ret; } @@ -342,117 +554,147 @@ static inline ssize_t pool_handle_write (pool_t* pool, int dst_fd) { - register int src_fd = pool->route_map[dst_fd]->sock; - pool_conn_end_t* dst = pool->route_map[src_fd]; + pool_conn_end_t* src = pool->route_map[dst_fd]; + pool_conn_end_t* dst = pool->route_map[src->sock]; + + glb_log_debug ("pool_handle_write() to %s: %zu", + dst->inc ? "client" : "server", dst->total - dst->sent); if (dst->total) { + ssize_t send_err; + assert (dst->total > dst->sent); - // if (dst->total == pool_buf_size), source was removed from fds_read - if (pool_send_data (pool, dst, (dst->total == pool_buf_size)) < 0) { - // probably don't care what error is - perror ("pool_handle_read(): sending data error"); + + if ((send_err = pool_send_data (pool, dst, src)) < 0) { + glb_log_warn ("pool_send_data(): %zd (%s)", + -send_err, strerror(-send_err)); } } return 0; } +// returns on error or after handling ctl - the latter may cause changes in +// file descriptors. +static inline long +pool_handle_events (pool_t* pool, long count) +{ + long idx; +#ifdef USE_EPOLL + for (idx = 0; idx < count; idx++) { + pollfd_t* pfd = pool->pollfds + idx; + if (pfd->events & POOL_FD_READ) { + + if (pfd->data.fd != pool->ctl_recv) { // normal read + register long ret; + pool->stats.poll_reads++; + + ret = pool_handle_read (pool, pfd->data.fd); + if (ret < 0) return ret; + } + else { // ctl read + return pool_handle_ctl (pool); + } + } + if (pfd->events & POOL_FD_WRITE) { + register long ret; + pool->stats.poll_writes++; + + assert (pfd->data.fd != pool->ctl_recv); + ret = pool_handle_write (pool, pfd->data.fd); + if (ret < 0) return ret; + } + } +#else // POLL + if (pool->pollfds[0].revents & POOL_FD_READ) { // first, check ctl socket + return pool_handle_ctl (pool); + } + + for (idx = 1; count > 0; idx++) + { + pollfd_t* pfd = pool->pollfds + idx; + + assert (idx < pool->fd_max); + + if (pfd->revents) { + // revents might be less than pfd->revents because some of the + // pfd->events might be cleared in the previous loop + register ulong revents = pfd->revents & pfd->events; + + if (revents & POOL_FD_READ) { + register long ret; + pool->stats.poll_reads++; + + ret = pool_handle_read (pool, pfd->fd); + if (ret < 0) return ret; + } + if (revents & POOL_FD_WRITE) { + register long ret; + pool->stats.poll_writes++; + + ret = pool_handle_write (pool, pfd->fd); + if (ret < 0) return ret; + } + count--; + } + } +#endif // POLL + return 0; +} + static void* pool_thread (void* arg) { pool_t* pool = arg; // synchronize with the calling process - pthread_mutex_lock (&pool->lock); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); while (1) { long ret; - fd_set fds_read, fds_write; - fds_read = pool->fds_read; - fds_write = pool->fds_write; - ret = select (pool->fd_max+1, &fds_read, &fds_write, NULL, NULL); - - if (ret > 0) { // we have some input - long count = ret; - int fd = pool->fd_min; - // first, check ctl pipe - if (FD_ISSET (pool->ctl_recv, &fds_read)) { - pool_ctl_t ctl; - - ret = read (pool->ctl_recv, &ctl, sizeof(ctl)); - if (sizeof(ctl) == ret) { // complete ctl read - pool_handle_ctl (pool, &ctl); - } - else { // should never happen! - perror ("Pool: incomplete read from ctl"); - abort(); - } -#ifdef GLB_POOL_STATS - pool->stats.sel_reads++; -#endif - /* - * pool->ctl_recv can theoretically get between fd_min and - * fd_max. - * For simplicity we want to assume that it is not set when - * processing normal fds - * also set of connections could be modified by ctl, - * so start over - */ - goto end; - } + ret = pool_fds_wait (pool); - assert (!FD_ISSET (pool->ctl_recv, &fds_read)); + if (ret > 0) { - // check remaining connections - while (count) { - assert (fd <= pool->fd_max); - - while (NULL == pool->route_map[fd]) fd++; - - /* - * If pool_handle_read() or pool_handle_write() below - * return error, this is most likely because connection was - * closed. In that case cleanup has happend and fd set has - * changed. Break out of the loop to start over again. - */ - if (FD_ISSET (fd, &fds_read)) { -#ifdef GLB_POOL_STATS - pool->stats.sel_reads++; -#endif - if (pool_handle_read (pool, fd) < 0) goto end; - count--; - } + pool->stats.n_polls++; - if (FD_ISSET (fd, &fds_write)) { -#ifdef GLB_POOL_STATS - pool->stats.sel_writes++; -#endif - if (pool_handle_write (pool, fd) < 0) goto end; - count--; - } + pool_handle_events (pool, ret); - fd++; - } } - else if (-1 == ret) { - perror ("select() failed"); + else if (ret < 0) { + glb_log_error ("pool_fds_wait() failed: %d (%s)", + errno, strerror(errno)); } else { - perror ("select() returned 0!"); - // timed out - //printf ("Thread %ld is idle\n", pool->id); + glb_log_error ("pool_fds_wait() interrupted: %d (%s)", + errno, strerror(errno)); } - end: -#ifdef GLB_POOL_STATS - pool->stats.n_select++; -#endif - continue; } return NULL; } +// initialize file descriptor set with ctl_recv descriptor +static long +pool_fds_init (pool_t* pool, int ctl_fd) +{ + pool->fd_max = 0; + +#ifdef USE_EPOLL + pool->epoll_fd = epoll_create(FD_SETSIZE); + if (pool->epoll_fd < 0) { + glb_log_fatal ("epoll_create(%d) failed: %d (%s)", + FD_SETSIZE, errno, strerror(errno)); + return -errno; + } +#endif + + pool->pollfds_len = 0; + + return pool_fds_add (pool, ctl_fd, POOL_FD_READ); +} + static long pool_init (pool_t* pool, long id, glb_router_t* router) { @@ -461,32 +703,35 @@ pool->id = id; pool->router = router; + pool->stats = glb_zero_stats; pthread_mutex_init (&pool->lock, NULL); pthread_cond_init (&pool->cond, NULL); ret = pipe(pipe_fds); if (ret) { - perror ("Failed to open control pipe"); + ret = errno; + glb_log_fatal ("Failed to open control pipe: %d (%s)", + ret, strerror(ret)); return -ret; } pool->ctl_recv = pipe_fds[0]; pool->ctl_send = pipe_fds[1]; - FD_ZERO (&pool->fds_read); - FD_ZERO (&pool->fds_write); - FD_SET (pool->ctl_recv, &pool->fds_read); - pool->fd_max = pool->ctl_recv; - pool->fd_min = pool->fd_max; + ret = pool_fds_init (pool, pool->ctl_recv); + if (ret < 0) { + return ret; + } - // this, together with pthread_mutex_lock() in the beginning of + // this, together with GLB_MUTEX_LOCK() in the beginning of // pool_thread() avoids possible race in access to pool->thread - pthread_mutex_lock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); ret = pthread_create (&pool->thread, NULL, pool_thread, pool); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); if (ret) { - perror ("Failed to create thread"); + glb_log_fatal ("Failed to create pool thread: %d (%s)", + ret, strerror(ret)); return -ret; } @@ -509,17 +754,19 @@ for (i = 0; i < n_pools; i++) { if ((err = pool_init(&ret->pool[i], i, router))) { - fprintf (stderr, "Failed to initialize pool %ld\n", i); + glb_log_fatal ("Failed to initialize pool %ld.", i); abort(); } } } else { - fprintf (stderr, "Could not allocate memory for %zu pools\n", n_pools); + glb_log_fatal ("Could not allocate memory for %zu pools.", + n_pools); abort(); } - gettimeofday (&ret->begin, NULL); + ret->last_info = glb_time_now(); + ret->last_stats = ret->last_info; return ret; } @@ -533,7 +780,7 @@ pthread_join (p->thread, NULL); } // TODO: proper resource deallocation - fprintf (stderr, "glb_pool_destroy() not implemented yet!"); + glb_log_warn ("glb_pool_destroy() not implemented yet!"); } // finds the least busy pool @@ -554,25 +801,25 @@ } // Sends ctl and waits for confirmation from the pool thread -extern ssize_t +static ssize_t pool_send_ctl (pool_t* p, pool_ctl_t* ctl) { ssize_t ret; - pthread_mutex_lock (&p->lock); + GLB_MUTEX_LOCK (&p->lock); ret = write (p->ctl_send, ctl, sizeof (*ctl)); if (ret != sizeof (*ctl)) { - perror ("Sending ctl failed"); + glb_log_error ("Sending ctl failed: %d (%s)", errno, strerror(errno)); if (ret > 0) abort(); // partial ctl was sent, don't know what to do } else ret = 0; pthread_cond_wait (&p->cond, &p->lock); - pthread_mutex_unlock (&p->lock); + GLB_MUTEX_UNLOCK (&p->lock); return ret; } -extern long +long glb_pool_add_conn (glb_pool_t* pool, int inc_sock, int dst_sock, @@ -582,15 +829,12 @@ long ret = -ENOMEM; void* route = NULL; - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_add_conn(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); route = malloc (pool_conn_size); if (route) { pool_conn_end_t* inc_end = route; - pool_conn_end_t* dst_end = route + pool_conn_size / 2; + pool_conn_end_t* dst_end = route + pool_end_size; pool_ctl_t add_conn_ctl = { POOL_CTL_ADD_CONN, route }; inc_end->inc = true; @@ -612,39 +856,75 @@ ret = pool_send_ctl (p, &add_conn_ctl); } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return ret; } -long -glb_pool_drop_dst (glb_pool_t* pool, const glb_sockaddr_t* dst) +// Sends the same ctl to all pools. Returns 0 minus how many ctls failed +static inline long +pool_bcast_ctl (glb_pool_t* pool, pool_ctl_t* ctl) { - pool_ctl_t drop_dst_ctl = { POOL_CTL_DROP_DST, (void*)dst }; ulong i; long ret = 0; - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_add_conn(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); for (i = 0; i < pool->n_pools; i++) { - ret -= (pool_send_ctl (&pool->pool[i], &drop_dst_ctl) < 0); + ret -= (pool_send_ctl (&pool->pool[i], ctl) < 0); } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return ret; } -size_t +long +glb_pool_drop_dst (glb_pool_t* pool, const glb_sockaddr_t* dst) +{ + pool_ctl_t drop_dst_ctl = { POOL_CTL_DROP_DST, (void*)dst }; + return pool_bcast_ctl (pool, &drop_dst_ctl); +} + +ssize_t glb_pool_print_stats (glb_pool_t* pool, char* buf, size_t buf_len) { - size_t len = 0; - long i; - struct timeval now; - double seconds; + glb_pool_stats_t stats = glb_zero_stats; + pool_ctl_t stats_ctl = { POOL_CTL_STATS, (void*)&stats }; + ssize_t ret; + glb_time_t now = glb_time_now(); + + ret = pool_bcast_ctl (pool, &stats_ctl); + if (!ret) { + double elapsed = now - pool->last_stats; + ret = snprintf (buf, buf_len, "in: %lu out: %lu " + "recv: %lu / %lu send: %lu / %lu " + "conns: %lu / %lu poll: %lu / %lu / %lu " + "elapsed: %.5f\n", + stats.rx_bytes, stats.tx_bytes, + stats.recv_bytes, stats.n_recv, + stats.send_bytes, stats.n_send, + stats.conns_opened, stats.n_conns, + stats.poll_reads, stats.poll_writes, stats.n_polls, + elapsed); + } + else { + assert (ret < 0); + glb_log_error ("Failed to get stats from %d thread pools.", ret); + } + + pool->last_stats = now; + + return ret; +} + +ssize_t +glb_pool_print_info (glb_pool_t* pool, char* buf, size_t buf_len) +{ + size_t len = 0; + long i; + glb_time_t now; + double elapsed; #ifndef GLB_POOL_STATS len += snprintf (buf + len, buf_len - len, "Pool: connections per thread:"); @@ -654,31 +934,28 @@ } #endif - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_print_stats(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); - gettimeofday (&now, NULL); - seconds = now.tv_sec - pool->begin.tv_sec + - (now.tv_usec - pool->begin.tv_usec) * 1.0e-06; + now = glb_time_now (); + elapsed = now - pool->last_info; for (i = 0; i < pool->n_pools; i++) { #ifdef GLB_POOL_STATS pool_stats_t s = pool->pool[i].stats; pool->pool[i].stats = zero_stats; + len += snprintf (buf + len, buf_len - len, "Pool %2ld: conns: %5ld, selects: %9zu (%9.2f sel/sec)\n" "recv : %9zuB %9zuR %9zuS %9.2fB/R %9.2fB/sec %9.2fR/S %9.2fR/sec\n" "send : %9zuB %9zuW %9zuS %9.2fB/W %9.2fB/sec %9.2fW/S %9.2fW/sec\n", - i, pool->pool[i].n_conns, s.n_select, (double)s.n_select/seconds, - s.recv_bytes,s.n_recv,s.sel_reads,(double)s.recv_bytes/s.n_recv, - (double)s.recv_bytes/seconds,(double)s.n_recv/s.n_select, - (double)s.n_recv/seconds, - s.send_bytes,s.n_send,s.sel_writes,(double)s.send_bytes/s.n_send, - (double)s.send_bytes/seconds,(double)s.n_send/s.n_select, - (double)s.n_send/seconds + i, pool->pool[i].n_conns, s.n_polls, (double)s.n_polls/elapsed, + s.recv_bytes,s.n_recv,s.poll_reads,(double)s.recv_bytes/s.n_recv, + (double)s.recv_bytes/elapsed,(double)s.n_recv/s.n_polls, + (double)s.n_recv/elapsed, + s.send_bytes,s.n_send,s.poll_writes,(double)s.send_bytes/s.n_send, + (double)s.send_bytes/elapsed,(double)s.n_send/s.n_polls, + (double)s.n_send/elapsed ); if (len == buf_len) { buf[len - 1] = '\0'; @@ -694,7 +971,7 @@ #endif } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); len += snprintf (buf + len, buf_len - len,"\n"); if (len == buf_len) { @@ -702,7 +979,7 @@ return (len - 1); } - pool->begin = now; + pool->last_info = now; return len; }

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_pool.h 23 2008-06-22 14:21:39Z alex $ + * $Id: glb_pool.h 59 2009-04-16 10:30:17Z alex $ */ #ifndef _glb_pool_h_ @@ -9,6 +9,7 @@ #include "glb_socket.h" #include "glb_router.h" +#include "glb_pool_stats.h" typedef struct glb_pool glb_pool_t; @@ -30,7 +31,10 @@ extern long glb_pool_drop_dst (glb_pool_t* pool, const glb_sockaddr_t* dst); -extern size_t +extern ssize_t glb_pool_print_stats (glb_pool_t* pool, char* buf, size_t buf_len); +extern ssize_t +glb_pool_print_info (glb_pool_t* pool, char* buf, size_t buf_len); + #endif // _glb_pool_h_

@@ -0,0 +1,46 @@ +/* + * Copyright (C) 2008 Codership Oy <info@codership.com> + * + * $Id: glb_pool_stats.h 59 2009-04-16 10:30:17Z alex $ + */ + +#ifndef _glb_pool_stats_h_ +#define _glb_pool_stats_h_ + +typedef struct glb_pool_stats +{ + ulong rx_bytes; // bytes received on incoming interface (from clients) + ulong tx_bytes; // bytes transmittted from incoming interface + ulong recv_bytes; // bytes received by recv() calls + ulong n_recv; // number of recv() calls + ulong send_bytes; // bytes sent by send() calls + ulong n_send; // number of send() calls + ulong conns_opened; // number of opened connections + ulong conns_closed; // number of closed connections + ulong n_conns; // number of current connections + ulong poll_reads; // number of read-ready fd's returned by poll() + ulong poll_writes; // number of write-ready fd's returned by poll() + ulong n_polls; // number of poll() calls +} glb_pool_stats_t; + +static const glb_pool_stats_t glb_zero_stats = { 0, }; + +// adds right stats to left stats +static inline void +glb_pool_stats_add (glb_pool_stats_t* left, glb_pool_stats_t* right) +{ + left->rx_bytes += right->rx_bytes; + left->tx_bytes += right->tx_bytes; + left->recv_bytes += right->recv_bytes; + left->n_recv += right->n_recv; + left->send_bytes += right->send_bytes; + left->n_send += right->n_send; + left->conns_opened += right->conns_opened; + left->conns_closed += right->conns_closed; + left->n_conns += right->n_conns; + left->poll_reads += right->poll_reads; + left->poll_writes += right->poll_writes; + left->n_polls += right->n_polls; +} + +#endif // _glb_pool_stats_h_

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_router.c 38 2008-07-15 18:03:19Z alex $ + * $Id: glb_router.c 59 2009-04-16 10:30:17Z alex $ */ #include <pthread.h> @@ -10,6 +10,7 @@ #include <errno.h> #include <stdio.h> #include <unistd.h> // for close() +#include <time.h> #include "glb_log.h" #include "glb_socket.h" @@ -20,10 +21,9 @@ typedef struct router_dst { glb_dst_t dst; - double weight; - long conns; // how many connections use this destination - double usage; // usage measure: weight/(conns + 1) - bigger wins - bool ready; // if destinaiton accepts connecitons + long conns; // how many connections use this destination + double usage; // usage measure: weight/(conns + 1) - bigger wins + time_t failed; // last time connection to this destination failed } router_dst_t; struct glb_router @@ -34,6 +34,12 @@ router_dst_t* dst; }; +static const double router_div_prot = 1.0e-09; +static inline double +router_dst_usage (router_dst_t* d) +{ return (d->dst.weight / (d->conns + router_div_prot)); } + + long glb_router_change_dst (glb_router_t* router, const glb_dst_t* dst) { @@ -71,8 +77,8 @@ else if (d->dst.weight != dst->weight) { // update weight and usage d->dst.weight = dst->weight; - d->weight = dst->weight; - d->usage = d->weight / (d->conns + 1); +// d->weight = dst->weight; + d->usage = router_dst_usage (d); } goto out; } @@ -98,10 +104,10 @@ d = router->dst + router->n_dst; router->n_dst++; d->dst = *dst; - d->weight = dst->weight; +// d->weight = dst->weight; d->conns = 0; - d->usage = d->weight / (d->conns + 1); - d->ready = true; + d->usage = router_dst_usage(d); + d->failed = 0; out: assert (router->n_dst >= 0); @@ -149,6 +155,9 @@ router_cleanup (router); } +// seconds (should be > 1 due to time_t precision) +static const double DST_RETRY_INTERVAL = 2.0; + // find a ready destination with minimal usage static router_dst_t* router_choose_dst (glb_router_t* router) @@ -158,10 +167,13 @@ if (router->n_dst > 0) { double max_usage = 0.0; int i; + time_t now = time(NULL); for (i = 0; i < router->n_dst; i++) { router_dst_t* d = &router->dst[i]; - if (d->ready && d->usage > max_usage) { + + if (d->usage > max_usage && + difftime (now, d->failed) > DST_RETRY_INTERVAL) { ret = d; max_usage = d->usage; } @@ -176,37 +188,28 @@ router_connect_dst (glb_router_t* router, int sock, glb_sockaddr_t* addr) { router_dst_t* dst; + int error = 1; + // keep trying until we run out of destinations while ((dst = router_choose_dst (router))) { if (connect (sock, (struct sockaddr*)&dst->dst.addr, sizeof (dst->dst.addr))) { - // connect failed, mark destination bad - fprintf (stderr, "Router: failed to connect to %s\n", - glb_socket_addr_to_string (&dst->dst.addr)); - dst->ready = false; - return -1; + error = errno; + // connect failed, update destination failed mark + fprintf (stderr, "Router: failed to connect to %s: %s\n", + glb_socket_addr_to_string (&dst->dst.addr), strerror(error)); + dst->failed = time(NULL); } else { // success, update stats dst->conns++; - dst->usage = dst->weight / (dst->conns + 1); + dst->usage = router_dst_usage(dst); *addr = dst->dst.addr; return 0; } } - return -1; -} - -// reset ready flag on destinations -// (TODO: associate a timestamp with a flag, don't reset right away) -static void -router_reset_dst (glb_router_t* router) -{ - long i; - for (i = 0; i < router->n_dst; i++) { - router->dst[i].ready = true; - } + return -error; // all attempts failed, return last errno } // returns error code @@ -219,7 +222,7 @@ sock = glb_socket_create (&router->sock_out); if (sock < 0) { perror ("Router: glb_socket_create"); - goto out; + return sock; } if (pthread_mutex_lock (&router->lock)) { @@ -229,7 +232,6 @@ // attmept to connect until we run out of destinations ret = router_connect_dst (router, sock, dst_addr); - router_reset_dst(router); // avoid socket leak if (ret < 0) { @@ -238,8 +240,8 @@ sock = ret; } -out: pthread_mutex_unlock (&router->lock); + return sock; } @@ -257,7 +259,7 @@ router_dst_t* d = &router->dst[i]; if (glb_socket_addr_is_equal (&d->dst.addr, dst)) { d->conns--; - d->usage = d->weight / (d->conns + 1); + d->usage = router_dst_usage(d); break; } } @@ -271,7 +273,7 @@ } size_t -glb_router_print_stats (glb_router_t* router, char* buf, size_t buf_len) +glb_router_print_info (glb_router_t* router, char* buf, size_t buf_len) { size_t len = 0; long total_conns = 0; @@ -296,9 +298,10 @@ total_conns += d->conns; - len += snprintf (buf + len, buf_len - len, "%s : %6ld %9.3f %6ld\n", + len += snprintf (buf + len, buf_len - len, "%s : %8.3f %7.3f %5ld\n", glb_socket_addr_to_string(&d->dst.addr), - d->dst.weight, 1.0 - (d->usage/d->weight), d->conns); + d->dst.weight, 1.0/(d->usage + 1.0), + d->conns); if (len == buf_len) { buf[len - 1] = '\0'; return (len - 1);

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_router.h 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_router.h 59 2009-04-16 10:30:17Z alex $ */ #ifndef _glb_router_h_ @@ -42,6 +42,6 @@ // Returns the length of the string extern size_t -glb_router_print_stats (glb_router_t* router, char* buf, size_t buf_len); +glb_router_print_info (glb_router_t* router, char* buf, size_t buf_len); #endif // _glb_router_h_

@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_socket.c 31 2008-06-30 17:17:14Z alex $ + * $Id: glb_socket.c 65 2009-06-06 21:16:51Z alex $ */ #include <stdio.h> @@ -12,9 +12,11 @@ #include <assert.h> #include <unistd.h> +#include "glb_log.h" #include "glb_socket.h" -//static const size_t addr_string_len = 512; heh, my GCC refuses to see it as a constant! here goes type safety... +//static const size_t addr_string_len = 512; heh, my GCC refuses to see it as +//a constant! here goes type safety... #define addr_string_len 512 static char addr_string[addr_string_len] = { 0, }; @@ -87,7 +89,8 @@ sock = socket (PF_INET, SOCK_STREAM, 0); if (sock < 0) { - perror ("glb_socket_create(): socket"); + glb_log_error ("Failed to create listening socket: %d (%s)", + errno, strerror (errno)); return -errno; } #if 0 @@ -107,7 +110,8 @@ /* Give the socket a name. */ if (bind (sock, (struct sockaddr *) addr, sizeof (*addr)) < 0) { - perror ("glb_socket_create(): bind"); + glb_log_error ("Failed to bind listening socket: %d (%s)", + errno, strerror (errno)); close (sock); return -errno; }

@@ -0,0 +1,22 @@ +/* + * Copyright (C) 2008 Codership Oy <info@codership.com> + * + * $Id: glb_time.h 57 2009-04-15 17:36:21Z alex $ + */ + +#ifndef _glb_time_h_ +#define _glb_time_h_ + +#include <sys/time.h> +#include <time.h> + +typedef double glb_time_t; + +static inline glb_time_t +glb_time_now() { + struct timeval tv; + gettimeofday (&tv, NULL); + return (1.0e-06 * tv.tv_usec + tv.tv_sec); +} + +#endif // _glb_time_h_

[-] [+]	Changed	glb.spec
@@ -1,12 +1,11 @@ Name: glb -Version: 0.7.1 +Version: 0.7.2 Release: 1 License: GPLv2 Group: Applications/System Summary: Galera Load Balancer -Source0: http://www.codership.com/files/glb/%{name}-%{version}.tar.gz -Patch0: glb-configure-%{version}.patch -Patch1: glb-pool-stats.patch +Source0: http://www.codership.com/files/glb/%{name}-%{version}.tar.bz2 +Patch0: glb-configure-0.7.1.patch BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: gcc-c++ @@ -26,16 +25,16 @@ %prep %setup %patch0 -%patch1 + %build -%configure \ - --enable-stats +%configure %__make %install %__make DESTDIR=%{buildroot} install +strip %{buildroot}%{_sbindir}/glbd %clean rm -rf %{buildroot} @@ -46,6 +45,11 @@ %{_sbindir}/glbd %changelog +* Sat Jun 20 2009 Carsten Schoene <cs@linux-administrator.com> - 0.7.2-1 +- update to release 0.7.2 + - Management through FIFO didn't work and has been fixed + - disabled --enable-stats (didn't compile since 0.7.1) + * Sun Apr 19 2009 Carsten Schoene <cs@linux-administrator.com> - 0.7.1-1 - update to release 0.7.1 - reworked configure patch (removes -Werror)
[-] [+]	Deleted	glb-pool-stats.patch ^
@@ -1,10 +0,0 @@ ---- src/glb_pool.c.orig 2009-04-19 21:18:55.000000000 +0200 -+++ src/glb_pool.c 2009-04-19 21:22:57.000000000 +0200 -@@ -31,6 +31,7 @@ - #include "glb_time.h" - #include "glb_log.h" - #include "glb_pool.h" -+#include "glb_pool_stats.h" - - extern bool glb_verbose; -
	Deleted	glb-0.6.3.tar.gz ^
	Deleted	glb-0.7.1.tar.gz ^
[-] [+]	Changed	glb-0.7.2.tar.bz2/ChangeLog ^
@@ -1,6 +1,47 @@ +2009-06-07 alex +- Fixed: + * Off-by-one bug which threw control thread into a tight loop when using FIFO + control method. + * listener thread was still using select(). + Version 0.7.2 + +2009-04-18 alex +- Fixed: + * control thread was still using select() to listen to requests (FD_SETSIZE + descriptor limit). + * reverted "fix" which opened control socket by default for security and + usability considerations. + Version 0.7.1. + +2009-04-16 alex +- Added machine parsable stats output on 'getstat' command +- Fixed: + * control socket was not opened by default + * usage display if weight was not 0 or 1 + * real numbers were not accepted for destination weight + Version 0.7.0. + +2009-04-15 alex +- Refactored to use poll/epoll API instead of select(). This allows for more + than FD_SETSIZE/2 concurrent connections and is considerably faster in case + of epoll. + +2009-03-12 alex +- Cosmetic changes to the output: usage now ranges strictly from 0.0 + (unused) to 1.0 (overloaded). Weight 0.0 automatically means usage 1.0, + before that it was 'nan'. Version 0.6.3. + +2009-02-10 alex +- Fixed non-working destination failover, failed destinations are retried + every 2 seconds. Version 0.6.2. + +2009-01-07 alex +- Fixed silly destination list parsing bug. Version 0.6.1. + 2008-09-14 alex - Fixed a problem with default port when adding destinations during runtime. - Changed from comma-separated destination list to space-separated destination list, hence bumped minor version. + 2008-07-15 alex - Initial release.
[-] [+]	Changed	glb-0.7.2.tar.bz2/README ^
@@ -19,8 +19,12 @@ - supports server "draining", i.e. does not allocate new connections to server, but does not kill existing ones, waiting for them to end gracefully. - - glb is multithreaded, so it can utilize multiple CPU cores. In fact even - on a single core CPU two threads may lower latencies a bit. + - can use epoll API provided by Linux version 2.6 and higher for ultimate + routing performance. + - glb is multithreaded, so it can utilize multiple CPU cores. Also, if your + OS does not support epoll API, consider using several threads even on a + single clore machine as it will lessen poll() overhead proportionally and + can improve overall performance by a factor of 2 or more. COMMAND LINE OPTIONS: ===================== @@ -49,12 +53,12 @@ OK $ echo "192.168.0.2:3307:5" \| nc -q 1 127.0.0.1 4444 OK -$ echo "getstat" \| nc -q 1 127.0.0.1 4444 +$ echo "getinfo" \| nc -q 1 127.0.0.1 4444 Router: ---------------------------------------------------- - Address : weight usage conns -192.168.0.1:3307 : 5 0.00 0 -192.168.0.2:3307 : 5 0.00 0 + Address : weight usage conns +192.168.0.1:3307 : 5.000 0.000 0 +192.168.0.2:3307 : 5.000 0.000 0 ---------------------------------------------------- Destinations: 2, total connections: 0 @@ -73,3 +77,36 @@ HOSTNAME1,HOSTNAME2,HOSTNAME3. In that case incoming port number will be used for PORT value and 1 will be used for WEIGHT value. + +PERFORMANCE STATISTICS: +======================= +GLB allows to query raw performance statistics through control socket using +'getstat' command. The client can use these data to obtain useful information, +e.g. average number of reads per poll() call. + +$ echo "getstat" \| nc -q 1 127.0.0.1 4444 +in: 6930 out: 102728 recv: 109658 / 45 send: 109658 / 45 conns: 0 / 4 poll: 45 / 0 / 45 elapsed: 1.03428 + +Statistics line consists of fields separated by spaces for ease of parsing in +scripts. A few description fields are added to assist in human reading. Value +fields are all even and go as follows: + + 2 - number of bytes received on incoming interface (client requests) + 4 - number of bytes sent from incoming inteface (server responses) + 6 - number of bytes passed through recv() call + 8 - number of recv() calls + 10 - number of bytes passed through send() call (should be equal to p.6) + 12 - number of send() calls + 14 - number of created connections + 16 - number of concurrent connections + 18 - number of read-ready file descriptors returned by poll()/epoll_wait() + 20 - number of write-ready file descriptors returned by poll()/epoll_wait() + 22 - number of times poll()/epoll_wait() triggered + 24 - time elapsed since last statistics report (seconds) + +All values except for 16 and 24 are totals accumulated since the last report. +In order to obtain some variable rate it must be divided by the elapsed time. +On 32-bit architectures the values are stored in 4-byte integers and can +overflow if too much time elapsed, so the first statistics report in the series +may need to be discarded. +
[-] [+]	Changed	glb-0.7.2.tar.bz2/config.h.in ^
@@ -19,6 +19,9 @@ /* Define to 1 if you have the `memset' function. / #undef HAVE_MEMSET +/ Define to 1 if you have the <poll.h> header file. / +#undef HAVE_POLL_H + / Define to 1 if your system has a GNU libc compatible `realloc' function, and to 0 otherwise. / #undef HAVE_REALLOC @@ -53,6 +56,9 @@ / Define to 1 if you have the `strtol' function. / #undef HAVE_STRTOL +/ Define to 1 if you have the <sys/epoll.h> header file. / +#undef HAVE_SYS_EPOLL_H + / Define to 1 if you have the <sys/stat.h> header file. */ #undef HAVE_SYS_STAT_H
[-] [+]	Changed	glb-0.7.2.tar.bz2/configure ^
@@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.61 for glb 0.6.0. +# Generated by GNU Autoconf 2.61 for glb 0.7.2. # # Report bugs to <info@codership.com>. # @@ -574,8 +574,8 @@ # Identity of this package. PACKAGE_NAME='glb' PACKAGE_TARNAME='glb' -PACKAGE_VERSION='0.6.0' -PACKAGE_STRING='glb 0.6.0' +PACKAGE_VERSION='0.7.2' +PACKAGE_STRING='glb 0.7.2' PACKAGE_BUGREPORT='info@codership.com' ac_unique_file="src/glb_main.c" @@ -686,10 +686,10 @@ AMTAR am__tar am__untar -ENABLE_DEBUG_TRUE -ENABLE_DEBUG_FALSE ENABLE_SPLICE_TRUE ENABLE_SPLICE_FALSE +ENABLE_DEBUG_TRUE +ENABLE_DEBUG_FALSE ENABLE_STATS_TRUE ENABLE_STATS_FALSE CXX @@ -1234,7 +1234,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures glb 0.6.0 to adapt to many kinds of systems. +\`configure' configures glb 0.7.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1305,15 +1305,18 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short \| recursive ) echo "Configuration of glb 0.6.0:";; + short \| recursive ) echo "Configuration of glb 0.7.2:";; esac cat <<\_ACEOF Optional Features: --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-debug enable debugging code [default=disabled] + --enable-poll use poll() for polling (default: use epoll() if + available) + --enable-splice use splice() [default=disabled] + --enable-debug enable debugging code [default=disabled] --enable-stats use extensive pool statistics [default=disabled] --disable-dependency-tracking speeds up one-time build --enable-dependency-tracking do not reject slow dependency extractors @@ -1394,7 +1397,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -glb configure 0.6.0 +glb configure 0.7.2 generated by GNU Autoconf 2.61 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1408,7 +1411,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by glb $as_me 0.6.0, which was +It was created by glb $as_me 0.7.2, which was generated by GNU Autoconf 2.61. Invocation command line was $ $0 $@ @@ -2228,7 +2231,7 @@ # Define the identity of the package. PACKAGE='glb' - VERSION='0.6.0' + VERSION='0.7.2' cat >>confdefs.h <<_ACEOF @@ -2377,27 +2380,12 @@ #AM_GNU_GETTEXT #AM_GNU_GETTEXT_VERSION(0.17) -# Check for debug -# Check whether --enable-debug was given. -if test "${enable_debug+set}" = set; then - enableval=$enable_debug; -else - enable_debug="no" -fi - -if test "$enable_debug" == "yes" -then - CFLAGS="-g -O1 -fno-inline " -else - CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} - CFLAGS="$CFLAGS -DNDEBUG " -fi - if test "$enable_debug" != "no"; then - ENABLE_DEBUG_TRUE= - ENABLE_DEBUG_FALSE='#' +# Check for poll method +# Check whether --enable-poll was given. +if test "${enable_poll+set}" = set; then + enableval=$enable_poll; else - ENABLE_DEBUG_TRUE='#' - ENABLE_DEBUG_FALSE= + enable_poll="no" fi @@ -2422,10 +2410,34 @@ fi +# Check for debug +# Check whether --enable-debug was given. +if test "${enable_debug+set}" = set; then + enableval=$enable_debug; +else + enable_debug="no" +fi + +if test "$enable_debug" == "yes" +then + CFLAGS="-g -O0 -fno-inline " +else + CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} + CFLAGS="$CFLAGS -DNDEBUG " +fi + if test "$enable_debug" != "no"; then + ENABLE_DEBUG_TRUE= + ENABLE_DEBUG_FALSE='#' +else + ENABLE_DEBUG_TRUE='#' + ENABLE_DEBUG_FALSE= +fi + + # Check for stats -# Check whether --enable-splice was given. -if test "${enable_splice+set}" = set; then - enableval=$enable_splice; +# Check whether --enable-stats was given. +if test "${enable_stats+set}" = set; then + enableval=$enable_stats; else enable_stats="no" fi @@ -4008,7 +4020,6 @@ #AC_PROG_LIBTOOL -# FIXME: Replace `main' with a function in `-lpthread': { echo "$as_me:$LINENO: checking for pthread_testcancel in -lpthread" >&5 @@ -6129,7 +6140,7 @@ -for ac_func in gettimeofday memset strcasecmp strdup strncasecmp strtol +for ac_func in gettimeofday memset strdup strcasecmp strncasecmp strtol do as_ac_var=`echo "ac_cv_func_$ac_func" \| $as_tr_sh` { echo "$as_me:$LINENO: checking for $ac_func" >&5 @@ -6222,6 +6233,492 @@ fi done + +# Check for poll()/epoll() + +for ac_header in poll.h +do +as_ac_Header=`echo "ac_cv_header_$ac_header" \| $as_tr_sh` +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + { echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } +else + # Is the header compilable? +{ echo "$as_me:$LINENO: checking $ac_header usability" >&5 +echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +$ac_includes_default +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" \|\| + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6; } + +# Is the header present? +{ echo "$as_me:$LINENO: checking $ac_header presence" >&5 +echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +#include <$ac_header> +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" \|\| + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5 +echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: $ac_header: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## --------------------------------- ## +## Report this to info@codership.com ## +## --------------------------------- ## +_ASBOX + ) \| sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + eval "$as_ac_Header=\$ac_header_preproc" +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } + +fi +if test `eval echo '${'$as_ac_Header'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_header" \| $as_tr_cpp` 1 +_ACEOF + { echo "$as_me:$LINENO: checking for poll" >&5 +echo $ECHO_N "checking for poll... $ECHO_C" >&6; } +if test "${ac_cv_func_poll+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +/ Define poll to an innocuous variant, in case <limits.h> declares poll. + For example, HP-UX 11i <limits.h> declares gettimeofday. / +#define poll innocuous_poll + +/ System header to define __stub macros and hopefully few prototypes, + which can conflict with char poll (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. / + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef poll + +/ Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. / +#ifdef __cplusplus +extern "C" +#endif +char poll (); +/ The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. / +#if defined __stub_poll \|\| defined __stub___poll +choke me +#endif + +int +main () +{ +return poll (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" \|\| + test ! -s conftest.err + } && test -s conftest$ac_exeext && + $as_test_x conftest$ac_exeext; then + ac_cv_func_poll=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_cv_func_poll=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +fi +{ echo "$as_me:$LINENO: result: $ac_cv_func_poll" >&5 +echo "${ECHO_T}$ac_cv_func_poll" >&6; } +if test $ac_cv_func_poll = yes; then + POLL="POLL" +else + { { echo "$as_me:$LINENO: error: * poll() not found! * +See \`config.log' for more details." >&5 +echo "$as_me: error: * poll() not found! * +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + +fi + + +else + { { echo "$as_me:$LINENO: error: * poll.h not found! * +See \`config.log' for more details." >&5 +echo "$as_me: error: * poll.h not found! * +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + +fi + +done + + +if test "$enable_poll" == "no" +then + +for ac_header in sys/epoll.h +do +as_ac_Header=`echo "ac_cv_header_$ac_header" \| $as_tr_sh` +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + { echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } +else + # Is the header compilable? +{ echo "$as_me:$LINENO: checking $ac_header usability" >&5 +echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +$ac_includes_default +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" \|\| + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6; } + +# Is the header present? +{ echo "$as_me:$LINENO: checking $ac_header presence" >&5 +echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +#include <$ac_header> +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" \|\| + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5 +echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: $ac_header: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: $ac_header: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## --------------------------------- ## +## Report this to info@codership.com ## +## --------------------------------- ## +_ASBOX + ) \| sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + eval "$as_ac_Header=\$ac_header_preproc" +fi +ac_res=`eval echo '${'$as_ac_Header'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } + +fi +if test `eval echo '${'$as_ac_Header'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_header" \| $as_tr_cpp` 1 +_ACEOF + { echo "$as_me:$LINENO: checking for epoll_create" >&5 +echo $ECHO_N "checking for epoll_create... $ECHO_C" >&6; } +if test "${ac_cv_func_epoll_create+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. / +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/ end confdefs.h. / +/ Define epoll_create to an innocuous variant, in case <limits.h> declares epoll_create. + For example, HP-UX 11i <limits.h> declares gettimeofday. / +#define epoll_create innocuous_epoll_create + +/ System header to define __stub macros and hopefully few prototypes, + which can conflict with char epoll_create (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. / + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef epoll_create + +/ Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. / +#ifdef __cplusplus +extern "C" +#endif +char epoll_create (); +/ The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. / +#if defined __stub_epoll_create \|\| defined __stub___epoll_create +choke me +#endif + +int +main () +{ +return epoll_create (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + \"* \| \` \| \\) ac_try_echo=\$ac_try;; + ) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ +' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" \|\| + test ! -s conftest.err + } && test -s conftest$ac_exeext && + $as_test_x conftest$ac_exeext; then + ac_cv_func_epoll_create=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/\| /' conftest.$ac_ext >&5 + + ac_cv_func_epoll_create=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +fi +{ echo "$as_me:$LINENO: result: $ac_cv_func_epoll_create" >&5 +echo "${ECHO_T}$ac_cv_func_epoll_create" >&6; } +if test $ac_cv_func_epoll_create = yes; then + POLL="EPOLL" +else + { echo "$as_me:$LINENO: WARNING: epoll API not found, trying poll()" >&5 +echo "$as_me: WARNING: epoll API not found, trying poll()" >&2;} + +fi + + + +fi + +done + +fi + if test "$enable_splice" == "yes" then @@ -6322,10 +6819,25 @@ # Many feature checks are broken and issue warnings. # If we want checks to pass we have to put this at the very end. -CFLAGS="$CFLAGS -Wall -Werror" +CFLAGS="$CFLAGS -Wall -Werror -DUSE_$POLL " -CXXFLAGS="$CFLAGS $CXXFLAGS" +#CXXFLAGS="$CFLAGS $CXXFLAGS" +#AC_SUBST(CXXFLAGS) +{ echo "$as_me:$LINENO: ----------------------" >&5 +echo "$as_me: ----------------------" >&6;} +{ echo "$as_me:$LINENO: poll method used: $POLL" >&5 +echo "$as_me: poll method used: $POLL" >&6;} +{ echo "$as_me:$LINENO: splice() enabled: $enable_splice" >&5 +echo "$as_me: splice() enabled: $enable_splice" >&6;} +{ echo "$as_me:$LINENO: stats enabled: $enable_stats" >&5 +echo "$as_me: stats enabled: $enable_stats" >&6;} +{ echo "$as_me:$LINENO: debug enabled: $enable_debug" >&5 +echo "$as_me: debug enabled: $enable_debug" >&6;} +{ echo "$as_me:$LINENO: CFLAGS = $CFLAGS" >&5 +echo "$as_me: CFLAGS = $CFLAGS" >&6;} +{ echo "$as_me:$LINENO: ----------------------" >&5 +echo "$as_me: ----------------------" >&6;} ac_config_files="$ac_config_files Makefile doc/Makefile src/Makefile" @@ -6425,17 +6937,17 @@ LTLIBOBJS=$ac_ltlibobjs -if test -z "${ENABLE_DEBUG_TRUE}" && test -z "${ENABLE_DEBUG_FALSE}"; then - { { echo "$as_me:$LINENO: error: conditional \"ENABLE_DEBUG\" was never defined. +if test -z "${ENABLE_SPLICE_TRUE}" && test -z "${ENABLE_SPLICE_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"ENABLE_SPLICE\" was never defined. Usually this means the macro was only invoked conditionally." >&5 -echo "$as_me: error: conditional \"ENABLE_DEBUG\" was never defined. +echo "$as_me: error: conditional \"ENABLE_SPLICE\" was never defined. Usually this means the macro was only invoked conditionally." >&2;} { (exit 1); exit 1; }; } fi -if test -z "${ENABLE_SPLICE_TRUE}" && test -z "${ENABLE_SPLICE_FALSE}"; then - { { echo "$as_me:$LINENO: error: conditional \"ENABLE_SPLICE\" was never defined. +if test -z "${ENABLE_DEBUG_TRUE}" && test -z "${ENABLE_DEBUG_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"ENABLE_DEBUG\" was never defined. Usually this means the macro was only invoked conditionally." >&5 -echo "$as_me: error: conditional \"ENABLE_SPLICE\" was never defined. +echo "$as_me: error: conditional \"ENABLE_DEBUG\" was never defined. Usually this means the macro was only invoked conditionally." >&2;} { (exit 1); exit 1; }; } fi @@ -6767,7 +7279,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by glb $as_me 0.6.0, which was +This file was extended by glb $as_me 0.7.2, which was generated by GNU Autoconf 2.61. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -6820,7 +7332,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -glb config.status 0.6.0 +glb config.status 0.7.2 configured by $0, generated by GNU Autoconf 2.61, with options \\"`echo "$ac_configure_args" \| sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" @@ -7072,10 +7584,10 @@ AMTAR!$AMTAR$ac_delim am__tar!$am__tar$ac_delim am__untar!$am__untar$ac_delim -ENABLE_DEBUG_TRUE!$ENABLE_DEBUG_TRUE$ac_delim -ENABLE_DEBUG_FALSE!$ENABLE_DEBUG_FALSE$ac_delim ENABLE_SPLICE_TRUE!$ENABLE_SPLICE_TRUE$ac_delim ENABLE_SPLICE_FALSE!$ENABLE_SPLICE_FALSE$ac_delim +ENABLE_DEBUG_TRUE!$ENABLE_DEBUG_TRUE$ac_delim +ENABLE_DEBUG_FALSE!$ENABLE_DEBUG_FALSE$ac_delim ENABLE_STATS_TRUE!$ENABLE_STATS_TRUE$ac_delim ENABLE_STATS_FALSE!$ENABLE_STATS_FALSE$ac_delim CXX!$CXX$ac_delim
[-] [+]	Changed	glb-0.7.2.tar.bz2/configure.ac ^
@@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.50) -AC_INIT(glb, 0.6.0, info@codership.com) +AC_INIT(glb, 0.7.2, info@codership.com) AC_CONFIG_SRCDIR([src/glb_main.c]) AC_CONFIG_HEADER([config.h]) AC_CANONICAL_SYSTEM @@ -11,6 +11,26 @@ #AM_GNU_GETTEXT #AM_GNU_GETTEXT_VERSION(0.17) +# Check for poll method +AC_ARG_ENABLE([poll], + [AC_HELP_STRING([--enable-poll], + [use poll() for polling (default: use epoll() if available)] + ) + ], + [], + enable_poll="no") + +# Check for splice() +AC_ARG_ENABLE(splice, + AC_HELP_STRING([--enable-splice], + [use splice() [[default=disabled]]]),, + enable_splice="no") +if test "$enable_splice" == "yes" +then + CFLAGS="$CFLAGS -D_GNU_SOURCE -DGLB_USE_SPLICE" +fi +AM_CONDITIONAL(ENABLE_SPLICE, test "$enable_splice" != "no") + # Check for debug AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug], @@ -18,26 +38,15 @@ enable_debug="no") if test "$enable_debug" == "yes" then - CFLAGS="-g -O1 -fno-inline " + CFLAGS="-g -O0 -fno-inline " else CFLAGS=${CFLAGS:-" -O3 -mtune=i686 "} CFLAGS="$CFLAGS -DNDEBUG " fi AM_CONDITIONAL(ENABLE_DEBUG, test "$enable_debug" != "no") -# Check for splice() -AC_ARG_ENABLE(splice, - AC_HELP_STRING([--enable-splice], - [use splice() [[default=disabled]]]),, - enable_splice="no") -if test "$enable_splice" == "yes" -then - CFLAGS="$CFLAGS -D_GNU_SOURCE -DGLB_USE_SPLICE" -fi -AM_CONDITIONAL(ENABLE_SPLICE, test "$enable_splice" != "no") - # Check for stats -AC_ARG_ENABLE(splice, +AC_ARG_ENABLE(stats, AC_HELP_STRING([--enable-stats], [use extensive pool statistics [[default=disabled]]]),, enable_stats="no") @@ -53,7 +62,6 @@ AC_PROG_CC #AC_PROG_LIBTOOL -# FIXME: Replace `main' with a function in `-lpthread': AC_CHECK_LIB([pthread], [pthread_testcancel],, AC_MSG_ERROR([* POSIX threads not found! ])) @@ -73,7 +81,28 @@ AC_FUNC_MALLOC AC_HEADER_STDC AC_FUNC_REALLOC -AC_CHECK_FUNCS([gettimeofday memset strcasecmp strdup strncasecmp strtol]) +AC_CHECK_FUNCS([gettimeofday memset strdup strcasecmp strncasecmp strtol]) + +# Check for poll()/epoll() +AC_CHECK_HEADERS([poll.h], + [AC_CHECK_FUNC([poll],[POLL="POLL"], + [AC_MSG_FAILURE([ poll() not found! ])] + ) + ], + [AC_MSG_FAILURE([ poll.h not found! *])] + ) + +if test "$enable_poll" == "no" +then + AC_CHECK_HEADERS([sys/epoll.h], + [AC_CHECK_FUNC([epoll_create], + [POLL="EPOLL"], + [AC_MSG_WARN([epoll API not found, trying poll()])] + ) + ] + ) +fi + if test "$enable_splice" == "yes" then AC_CHECK_FUNCS([splice]) @@ -81,10 +110,18 @@ # Many feature checks are broken and issue warnings. # If we want checks to pass we have to put this at the very end. -CFLAGS="$CFLAGS -Wall -Werror" +CFLAGS="$CFLAGS -Wall -Werror -DUSE_$POLL " AC_SUBST(CFLAGS) -CXXFLAGS="$CFLAGS $CXXFLAGS" -AC_SUBST(CXXFLAGS) +#CXXFLAGS="$CFLAGS $CXXFLAGS" +#AC_SUBST(CXXFLAGS) + +AC_MSG_NOTICE([----------------------]) +AC_MSG_NOTICE([poll method used: $POLL]) +AC_MSG_NOTICE([splice() enabled: $enable_splice]) +AC_MSG_NOTICE([stats enabled: $enable_stats]) +AC_MSG_NOTICE([debug enabled: $enable_debug]) +AC_MSG_NOTICE([CFLAGS = $CFLAGS]) +AC_MSG_NOTICE([----------------------]) AC_CONFIG_FILES([Makefile doc/Makefile
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_cmd.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_cmd.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_cmd.c 63 2009-04-18 12:09:59Z alex $ / #include <stddef.h> // ptrdiff_t @@ -51,9 +51,8 @@ }; // Some constants -static const int cmd_list_separator = ','; -static const long cmd_ip_len_max = 256; -static const long cmd_port_max = (1<<16) - 1; +#define cmd_ip_len_max 256 +#define cmd_port_max ((1<<16) - 1) void glb_cmd_help (FILE out, const char* progname) @@ -62,32 +61,30 @@ "Usage:\n %s [OPTIONS] LISTEN_ADDRESS " "[DESTINATION_LIST]\nOPTIONS:\n", progname); fprintf (out, - " --help this help message\n"); + " --help this help message.\n"); fprintf (out, - " --fifo <fifo name> name of the FIFO file for control\n"); + " --fifo <fifo name> name of the FIFO file for control.\n"); fprintf (out, " --control [HOST:]PORT " - "listen for control requests on this address\n" - " " - "(default: 127.0.0.1:<LISTEN_PORT + 1>\n"); + "listen for control requests on this address.\n"); fprintf (out, " --threads N " - "number of working threads (number of CPU cores)\n"); + "number of working threads (connection pools).\n"); fprintf (out, " --source_tracking " "turn on source tracking: route connections from one\n" - " source to the same destination\n"); + " source to the same destination.\n"); fprintf (out, - " --verbose turn on verbose reporting\n"); + " --verbose turn on verbose reporting.\n"); fprintf (out, - " --version print program version\n"); + " --version print program version.\n"); fprintf (out, "LISTEN_ADDRESS:\n" " [IP:]PORT " - "where to listen for incoming TCP connections\n"); + "where to listen for incoming TCP connections.\n"); fprintf (out, "DESTINATION_LIST:\n" " [H1[:P1[:W1]]] [H2[:P2[:W2]]]... " " - a space-separated list of destinations\n" - " in the form address:port:weight\n"); + " in the form address:port:weight.\n"); exit (EXIT_FAILURE); } @@ -99,7 +96,7 @@ fprintf (out, "Incoming address: %s, ", glb_socket_addr_to_string (&cmd->inc_addr)); fprintf (out, "control FIFO: %s\n", cmd->fifo_name); - fprintf (out, "Control address: %s\n", + fprintf (out, "Control address: %s\n", cmd->ctrl_set ? glb_socket_addr_to_string (&cmd->ctrl_addr) : "none"); fprintf (out, "Number of threads: %lu, source tracking: %s, verbose: %s, " @@ -157,7 +154,7 @@ return glb_socket_addr_init (addr, addr_str, port); } -// parses comma separated list of destinations, reallocates and fills conf +// parses array list of destinations static glb_cmd_t* cmd_parse_dst_list (const char* dst_list[], size_t n_dst, @@ -165,25 +162,12 @@ { glb_cmd_t* ret = NULL; size_t i; - const size_t max_dst_len = 256; // addr:port:weight\0 - char dst_str[max_dst_len + 1] = { 0, }; - ptrdiff_t dst_len; ret = calloc (sizeof(ret) + n_dst sizeof(glb_dst_t), 1); if (ret) { for (i = 0; i < n_dst; i++) { - - dst_len = strlen (dst_list[i]); - - if (dst_len > max_dst_len) { - fprintf (stderr, "Destination spec too long: %s\n",dst_list[i]); - free (ret); - return NULL; - } - - strncpy (dst_str, dst_list[i], dst_len); - - switch (glb_dst_parse (&ret->dst[i], dst_str, default_port)) { + + switch (glb_dst_parse (&ret->dst[i], dst_list[i], default_port)) { case 1: // default port is assigned glb_dst_parse() case 2: @@ -191,7 +175,7 @@ case 3: break; default: // error parsing destination - fprintf (stderr, "Invalid destination spec: %s\n", dst_str); + fprintf (stderr, "Invalid destination spec: %s\n", dst_list[i]); free (ret); return NULL; } @@ -282,6 +266,17 @@ } inc_port = glb_socket_addr_get_port (&tmp.inc_addr); +#if 0 // don't open socket by default for security considerations. + // if control address was not specified + if (!tmp.ctrl_set) { + char port[6] = { 0, }; + snprintf (port, 5, "%hu", inc_port + 1); + if (cmd_parse_addr (&tmp.ctrl_addr, port, cmd_ctrl_addr_default)) + return NULL; + tmp.ctrl_set = true; + } +#endif + // if number of threads was not specified if (!tmp.n_threads) tmp.n_threads = 1;
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_control.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_control.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_control.c 65 2009-06-06 21:16:51Z alex $ / // keep asserts here for now @@ -16,20 +16,31 @@ #include <errno.h> #include <stdlib.h> #include <string.h> -#include <sys/socket.h> -#include <sys/un.h> #include <sys/stat.h> #include <fcntl.h> #include <ctype.h> +// unfotunately I see no way to use glb_pool.c polling code in here +// so it is yet another implementation +#include <poll.h> +typedef struct pollfd pollfd_t; + #include "glb_log.h" #include "glb_signal.h" #include "glb_control.h" extern bool glb_verbose; +static const char ctrl_getinfo_cmd[] = "getinfo"; static const char ctrl_getstat_cmd[] = "getstat"; +typedef enum ctrl_fd +{ + CTRL_FIFO = 0, + CTRL_LISTEN, + CTRL_MAX = 32 // max 30 simultaneous control connections +} ctrl_fd_t; + struct glb_ctrl { pthread_t thread; @@ -39,28 +50,51 @@ glb_router_t router; glb_pool_t* pool; int fd_max; - fd_set fds; + pollfd_t fds[CTRL_MAX]; uint16_t default_port; }; static void ctrl_add_client (glb_ctrl_t* ctrl, int fd) { - assert (ctrl->fd_max != fd); - FD_SET (fd, &ctrl->fds); - if (fd > ctrl->fd_max) ctrl->fd_max = fd; + assert (ctrl->fd_max < CTRL_MAX); + + ctrl->fds[ctrl->fd_max].fd = fd; + ctrl->fds[ctrl->fd_max].events = POLLIN; + ctrl->fds[ctrl->fd_max].revents = 0; + + ctrl->fd_max++; + + if (CTRL_MAX == ctrl->fd_max) // no more clients + ctrl->fds[CTRL_LISTEN].events = 0; } static void ctrl_del_client (glb_ctrl_t* ctrl, int fd) { - assert (ctrl->fd_max >= fd); - FD_CLR (fd, &ctrl->fds); - if (fd == ctrl->fd_max) { // find next highest fd - while (!FD_ISSET (ctrl->fd_max, &ctrl->fds)) ctrl->fd_max--; + int idx; + + assert (CTRL_MAX >= ctrl->fd_max); + assert (ctrl->fd_max > CTRL_LISTEN); + + for (idx = 1; idx < ctrl->fd_max; idx++) { + if (fd == ctrl->fds[idx].fd) { + close (ctrl->fds[idx].fd); + ctrl->fd_max--; + ctrl->fds[idx] = ctrl->fds[ctrl->fd_max]; + ctrl->fds[ctrl->fd_max].fd = 0; + ctrl->fds[ctrl->fd_max].events = 0; + ctrl->fds[ctrl->fd_max].revents = 0; + break; + } } - assert (ctrl->fd_max < fd); - close (fd); + + if (!(ctrl->fd_max < CTRL_MAX)) { + glb_log_fatal ("Failed to cleanup control connection."); + abort(); + }; + + ctrl->fds[CTRL_LISTEN].events = POLLIN; } static inline void @@ -104,8 +138,13 @@ } } - if (!strncasecmp (ctrl_getstat_cmd, req, strlen(ctrl_getstat_cmd))) { - glb_router_print_stats (ctrl->router, req, sizeof(req)); + if (!strncasecmp (ctrl_getinfo_cmd, req, strlen(ctrl_getinfo_cmd))) { + glb_router_print_info (ctrl->router, req, sizeof(req)); + ctrl_respond (ctrl, fd, req); + return 0; + } + else if (!strncasecmp (ctrl_getstat_cmd, req, strlen(ctrl_getstat_cmd))) { + glb_pool_print_stats (ctrl->pool, req, sizeof(req)); ctrl_respond (ctrl, fd, req); return 0; } @@ -126,38 +165,50 @@ ctrl_respond (ctrl, fd, "Error\n"); return 0; } - ctrl_respond (ctrl, fd, "OK\n"); - if (dst.weight < 0) { + ctrl_respond (ctrl, fd, "Ok\n"); + + if (dst.weight < 0.0) { // destination was removed from router, drop all connections to it glb_pool_drop_dst (ctrl->pool, &dst.addr); } + return 0; } } +// returns true if fd is ready to read +static inline bool +ctrl_fd_isset (glb_ctrl_t* ctrl, int idx) +{ + return (ctrl->fds[idx].revents & POLLIN); +} + static void* ctrl_thread (void* arg) { glb_ctrl_t* ctrl = arg; while (!glb_terminate) { - long ret; - int client_sock; - struct sockaddr client; - socklen_t client_size; - fd_set fds = ctrl->fds; - struct timeval timeout = { 1, 0 }; + long ret; - ret = select (ctrl->fd_max + 1, &fds, NULL, NULL, &timeout); + // Timeout is needed to gracefully shut down + ret = poll (ctrl->fds, ctrl->fd_max, 1000 /* ms */); if (ret < 0) { - perror ("error waiting for connections"); + glb_log_error ("Error waiting for connections: %d (%s)", + errno, strerror(errno)); goto err; //? } else if (0 == ret) continue; - if (ctrl->inet_sock > 0 && FD_ISSET (ctrl->inet_sock, &fds)) { + if (ctrl->inet_sock > 0 && (ctrl_fd_isset (ctrl, CTRL_LISTEN))) { + // new network client + int client_sock; + struct sockaddr client; + socklen_t client_size; + client_sock = accept (ctrl->inet_sock, &client, &client_size); + if (client_sock < 0) { glb_log_error ("Ctrl: failed to accept connection: %d (%s)", errno, strerror (errno)); @@ -175,10 +226,12 @@ } else { int fd; - for (fd = 1; fd <= ctrl->fd_max; fd++) { // find fd - if (FD_ISSET (fd, &fds)) break; + for (fd = CTRL_FIFO; fd <= ctrl->fd_max; fd++) { // find fd + if (ctrl_fd_isset (ctrl, fd)) { + assert (fd != CTRL_LISTEN); + if (ctrl_handle_request (ctrl, ctrl->fds[fd].fd)) goto err; + } } - if (ctrl_handle_request (ctrl, fd)) goto err; continue; } @@ -247,21 +300,17 @@ ret->fifo_name = fifo_name; ret->fifo = fifo; ret->inet_sock = inet_sock; - ret->fd_max = fifo > inet_sock ? fifo : inet_sock; + ret->fd_max = fifo > inet_sock ? 1 : 2; ret->default_port = port; - FD_ZERO (&ret->fds); - FD_SET (ret->fifo, &ret->fds); - if (ret->inet_sock > 0) FD_SET (ret->inet_sock, &ret->fds); - -#if 0 // remove - for (fd = 0; fd < FD_SETSIZE; fd++) { - if (FD_ISSET (fd, &ret->fds)) - printf ("Ctrl: set fd %d\n", fd); - } - printf ("Ctrl: fd_max = %d\n", ret->fd_max); - printf ("Ctrl: ctrl object: %p\n", ret); -#endif + ret->fds[CTRL_FIFO].fd = ret->fifo; + ret->fds[CTRL_FIFO].events = POLLIN; + + if (ret->inet_sock > 0) { + ret->fds[CTRL_LISTEN].fd = ret->inet_sock; + ret->fds[CTRL_LISTEN].events = POLLIN; + } + if (pthread_create (&ret->thread, NULL, ctrl_thread, ret)) { glb_log_error ("Failed to launch ctrl thread."); free (ret);
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_dst.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_dst.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_dst.c 59 2009-04-16 10:30:17Z alex $ / #include <stddef.h> // ptrdiff_t @@ -12,11 +12,11 @@ #include "glb_dst.h" // Some constants -static const int dst_separator = ':'; -static const ptrdiff_t dst_ip_len_max = 256; -static const ptrdiff_t dst_ip_len_min = 1; -static const ulong dst_port_max = (1 << 16) - 1; -static const long dst_default_weight = 1; +#define dst_separator ':' +#define dst_ip_len_max 256 +#define dst_ip_len_min 1 +#define dst_port_max ((1 << 16) - 1) +#define dst_default_weight 1.0 // parses addr:port:weight string, stores in dst // returns number of parsed fields or negative error code @@ -72,7 +72,7 @@ // parse weight assert (endptr == dst_separator); token = endptr + 1; - dst->weight = strtoul (token, &endptr, 10); + dst->weight = strtod (token, &endptr); if (*endptr != '\0') { perror ("Weight field doesn't consist only of numbers"); return -EINVAL;
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_dst.h ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_dst.h 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_dst.h 59 2009-04-16 10:30:17Z alex $ / #ifndef _glb_dst_h_ @@ -16,9 +16,9 @@ typedef struct glb_dst { glb_sockaddr_t addr; // destination address in prepared form - long weight; // >0: connection allocation weight (def: 1) + double weight; // >0: connection allocation weight (def: 1) // 0: no new conns, but keep existing (drain) - // -1: discard destination entirely + // <0: discard destination entirely } glb_dst_t; /! @@ -35,7 +35,7 @@ } static inline void -glb_dst_set_weight (glb_dst_t* dst, long weight) +glb_dst_set_weight (glb_dst_t* dst, double weight) { dst->weight = weight; } @@ -49,7 +49,7 @@ static inline void glb_dst_print (char* buf, size_t buf_len, const glb_dst_t* dst) { - snprintf (buf, buf_len, "%s, w: %ld", + snprintf (buf, buf_len, "%s, w: %5.3f", glb_socket_addr_to_string(&dst->addr), dst->weight); buf[buf_len - 1] = '\0'; }
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_listener.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_listener.c 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_listener.c 65 2009-06-06 21:16:51Z alex $ / #include <pthread.h> @@ -9,6 +9,8 @@ #include <unistd.h> #include <errno.h> #include <string.h> +#include <poll.h> +typedef struct pollfd pollfd_t; #include "glb_log.h" #include "glb_listener.h" @@ -27,11 +29,7 @@ listener_thread (void arg) { glb_listener_t* listener = arg; - fd_set fds; - - // shoudl be no need to refresh the fd set - FD_ZERO (&fds); - FD_SET (listener->sock, &fds); + pollfd_t pollfd = { .fd = listener->sock, .events = POLLIN, .revents = 0 }; while (1) { long ret; @@ -41,28 +39,27 @@ int server_sock; glb_sockaddr_t server; - ret = select (listener->sock + 1, &fds, NULL, NULL, NULL); + ret = poll (&pollfd, 1, -1); if (ret < 0) { - glb_log_error ("error waiting for connections: %d (%s)", + glb_log_error ("Error waiting for connections: %d (%s)", errno, strerror (errno)); - FD_SET (listener->sock, &fds); goto err; //? } assert (1 == ret); - assert (FD_ISSET (listener->sock, &fds)); + assert (pollfd.revents & POLLIN); client_sock = accept (listener->sock, (struct sockaddr*) &client, &client_size); if (client_sock < 0) { - glb_log_error ("Listener: failed to accept connection: %d (%s)", + glb_log_error ("Failed to accept connection: %d (%s)", errno, strerror (errno)); goto err; } server_sock = glb_router_connect (listener->router, &server); if (server_sock < 0) { - glb_log_error("Listener: failed to connect to destination: %d (%s)", + glb_log_error("Failed to connect to destination: %d (%s)", errno, strerror(errno)); goto err1; } @@ -70,16 +67,16 @@ ret = glb_pool_add_conn (listener->pool, client_sock, server_sock, &server); if (ret < 0) { - glb_log_error ("Listener: failed to add connection to pool: " + glb_log_error ("Failed to add connection to pool: " "%d (%s)", errno, strerror (errno)); goto err2; } if (glb_verbose) { - fprintf (stderr, "Listener: accepted connection from %s ", - glb_socket_addr_to_string (&client)); - fprintf (stderr, "to %s\n", - glb_socket_addr_to_string (&server)); + glb_log_info ("Accepted connection from %s ", + glb_socket_addr_to_string (&client)); + glb_log_info ("to %s\n", + glb_socket_addr_to_string (&server)); } continue;
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_log.h ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_log.h 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_log.h 57 2009-04-15 17:36:21Z alex $ */ #ifndef _glb_log_h_ @@ -51,9 +51,12 @@ glb_log (GLB_LOG_INFO, __FILE__, __PRETTY_FUNCTION__, __LINE__,\ format, ## __VA_ARGS__, NULL) +#ifdef NDEBUG +#define glb_log_debug(format, ...) +#else // DEBUG #define glb_log_debug(format, ...) \ glb_log (GLB_LOG_DEBUG, __FILE__, __PRETTY_FUNCTION__, __LINE__,\ format, ## __VA_ARGS__, NULL) - +#endif // DEBUG #endif // _glb_log_h_
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_main.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_main.c 41 2008-09-14 12:18:24Z alex $ + * $Id: glb_main.c 59 2009-04-16 10:30:17Z alex $ */ #include <unistd.h> // for sleep() @@ -85,10 +85,10 @@ if (!cmd->daemonize) { char stats[BUFSIZ]; - glb_router_print_stats (router, stats, BUFSIZ); + glb_router_print_info (router, stats, BUFSIZ); puts (stats); - glb_pool_print_stats (pool, stats, BUFSIZ); + glb_pool_print_info (pool, stats, BUFSIZ); puts (stats); }
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_pool.c ^
@@ -1,11 +1,9 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_pool.c 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_pool.c 61 2009-04-17 22:23:24Z alex $ / -#include <stdlib.h> -#include <sys/select.h> // for select() and FD_SET #include <pthread.h> #include <stdio.h> #include <string.h> @@ -13,12 +11,25 @@ #include <unistd.h> #include <errno.h> #include <assert.h> -#include <sys/time.h> + +#ifndef USE_EPOLL + #ifndef USE_POLL + #error "Neither USE_POLL nor USE_EPOLL defined!" + #else + #include <poll.h> + typedef struct pollfd pollfd_t; + #endif +#else + #include <sys/epoll.h> + typedef struct epoll_event pollfd_t; +#endif #ifdef GLB_USE_SPLICE #include <fcntl.h> #endif +#include "glb_time.h" +#include "glb_log.h" #include "glb_pool.h" extern bool glb_verbose; @@ -28,6 +39,7 @@ POOL_CTL_ADD_CONN, POOL_CTL_DROP_DST, POOL_CTL_SHUTDOWN, + POOL_CTL_STATS, POOL_CTL_MAX } pool_ctl_code_t; @@ -37,25 +49,12 @@ void data; } pool_ctl_t; -typedef struct pool_stats -{ - size_t recv_bytes; - size_t n_recv; - size_t send_bytes; - size_t n_send; - size_t sel_reads; - size_t sel_writes; - size_t n_select; -} pool_stats_t; - -#ifdef GLB_POOL_STATS -static pool_stats_t zero_stats = { 0, }; -#endif - typedef struct pool_conn_end { bool inc; // to differentiate between the ends int sock; // fd of connection + int fds_idx; // index in the file descriptor set (for poll()) + uint32_t events; // events waited by descriptor size_t sent; size_t total; glb_sockaddr_t dst_addr; // destinaiton id @@ -65,79 +64,208 @@ uint8_t buf[]; // has pool_buf_size } pool_conn_end_t; +#define POOL_MAX_FD (1 << 16) // highest possible file descriptor + 1 + // only affects the map size + // We want to allocate memory for both ends in one malloc() call and have it // nicely aligned. This is presumably a page multiple and // should be enough for two ethernet frames (what about jumbo?) -#define pool_conn_size (BUFSIZ << 2) -const size_t pool_buf_size = (pool_conn_size/2 - sizeof(pool_conn_end_t)); +#define pool_end_size (BUFSIZ) +#define pool_conn_size (pool_end_size << 2) +#define pool_buf_size (pool_end_size - sizeof(pool_conn_end_t)) typedef struct pool { - long id; - pthread_t thread; - pthread_mutex_t lock; - pthread_cond_t cond; - int ctl_recv; // receive commands - pool thread - int ctl_send; // send commands to pool - other function - volatile ulong n_conns; // how many connecitons this pool serves - fd_set fds_read; - fd_set fds_write; - int fd_max; - int fd_min; - glb_router_t* router; -#ifdef GLB_POOL_STATS - volatile pool_stats_t stats; -#endif - pool_conn_end_t* route_map[FD_SETSIZE]; // looking for connection ctx by fd + long id; + pthread_t thread; + pthread_mutex_t lock; + pthread_cond_t cond; + int ctl_recv; // fd to receive commands in pool thread + int ctl_send; // fd to send commands to pool - other function + volatile ulong n_conns; // how many connecitons this pool serves +#ifdef USE_EPOLL + int epoll_fd; +#endif + pollfd_t* pollfds; + size_t pollfds_len; + int fd_max; + glb_router_t* router; + glb_pool_stats_t stats; + + pool_conn_end_t* route_map[ POOL_MAX_FD ]; // connection ctx look-up by fd } pool_t; struct glb_pool { pthread_mutex_t lock; ulong n_pools; - struct timeval begin; + glb_time_t last_info; + glb_time_t last_stats; pool_t pool[]; // pool array, can't be changed in runtime }; +typedef enum pool_fd_ops +{ +#ifdef USE_EPOLL + POOL_FD_READ = EPOLLIN, + POOL_FD_WRITE = EPOLLOUT, +#else // POLL + POOL_FD_READ = POLLIN, + POOL_FD_WRITE = POLLOUT, +#endif // POLL + POOL_FD_RW = POOL_FD_READ \| POOL_FD_WRITE +} pool_fd_ops_t; + +//#define FD_SETSIZE 1024; // leater get it from select.h + +static const pollfd_t zero_pollfd = { 0, }; + +/! + @return negative error code or the index of file descriptor in the set + / +static inline long +pool_fds_add (pool_t pool, int fd, pool_fd_ops_t events) +{ + int ret; + + assert (fd > 0); + assert (pool->fd_max <= pool->pollfds_len); + + if (pool->fd_max == pool->pollfds_len) { // allocate more memory + void* tmp; + size_t tmp_len = pool->pollfds_len + FD_SETSIZE; + + tmp = realloc (pool->pollfds, tmp_len * sizeof(pollfd_t)); + if (NULL == tmp) { + glb_log_fatal ("Failed to (re)allocate %d pollfds: out of memory", + tmp_len); + return -ENOMEM; + } + + memset (((pollfd_t)tmp) + pool->fd_max, 0, + (tmp_len - pool->fd_max) sizeof(pollfd_t)); + + pool->pollfds = tmp; + pool->pollfds_len = tmp_len; + } + +#ifdef USE_EPOLL + struct epoll_event add_event = { .events = events, { .fd = fd }}; + + ret = epoll_ctl (pool->epoll_fd, EPOLL_CTL_ADD, fd, &add_event); + if (ret) { + glb_log_error ("epoll_ctl (%d, EPOLL_CTL_ADD, %d, {%d, %llu}) failed: " + "%d (%s)", + pool->epoll_fd, fd, add_event.events, add_event.data.u64, + errno, strerror (errno)); + return -errno; + } +#else // POLL + pool->pollfds[pool->fd_max].fd = fd; + pool->pollfds[pool->fd_max].events = events; +#endif // POLL + + ret = pool->fd_max; + + pool->fd_max++; // track how many descriptors are there + + return ret; +} + +// returns corresponding pool_conn_end_t* +static inline pool_conn_end_t* +pool_conn_end_by_fd (pool_t* pool, int fd) +{ + // map points to the other end, but that's enough + register pool_conn_end_t* other_end = pool->route_map[fd]; + if (other_end->inc) { + return (pool_conn_end_t)((uint8_t)other_end + pool_end_size); + } else { + return (pool_conn_end_t)((uint8_t)other_end - pool_end_size); + } +} + +// remove file descriptor from file descriptor set +static inline long +pool_fds_del (pool_t* pool, pool_conn_end_t* end) +{ + pool->fd_max--; // pool->fd_max is now the index of the last pollfd + +#ifdef USE_EPOLL + long ret = epoll_ctl (pool->epoll_fd, EPOLL_CTL_DEL, end->sock, NULL); + if (ret) { + glb_log_error ("epoll_ctl (%d, EPOLL_CTL_DEL, %d, NULL) failed: " + "%d (%s)", + pool->epoll_fd, end->sock, errno, strerror (errno)); + return -errno; + } +#else // POLL + assert (end->fds_idx <= pool->fd_max); + + /* + * pay attention here: the last pollfd that we're moving may have not been + * checked yet + / + + // copy the last pollfd in place of the deleted + pool->pollfds[end->fds_idx] = pool->pollfds[pool->fd_max]; + // from this pollfd find its fd and from route_map by fd find its + // pool_conn_end struct and in that struct update fds_idx to point at + // a new position. + pool_conn_end_by_fd(pool, pool->pollfds[end->fds_idx].fd)->fds_idx = + end->fds_idx; + // zero-up the last pollfd + pool->pollfds[pool->fd_max] = zero_pollfd; +#endif // POLL + + return 0; +} + +static inline void +pool_fds_set_events (pool_t pool, pool_conn_end_t* end) +{ +#ifdef USE_EPOLL + struct epoll_event event = { .events = end->events, { .fd = end->sock } }; + if (epoll_ctl (pool->epoll_fd, EPOLL_CTL_MOD, end->sock, &event)) { + glb_log_fatal ("epoll_ctl(%d, EPOLL_CTL_MOD, %d, {%d, %llu}) failed: " + "%d (%s)", + pool->epoll_fd, end->sock, event.events, event.data.u64, + errno, strerror(errno)); + abort(); + } +#else // POLL + pool->pollfds[end->fds_idx].events = end->events; +#endif // POLL +} + +static inline long +pool_fds_wait (pool_t* pool) +{ +#ifdef USE_EPOLL + return epoll_wait (pool->epoll_fd, pool->pollfds, pool->fd_max, -1); +#else // POLL + return poll (pool->pollfds, pool->fd_max, -1); +#endif // POLL +} + // performs necessary magic (adds end-to-end mapping, alters fd_max and fd_min) // when new file descriptor is added to fd_set static inline void pool_set_conn_end (pool_t* pool, pool_conn_end_t* end1, pool_conn_end_t* end2) { - assert (end1->sock < FD_SETSIZE); - assert (NULL == pool->route_map[end1->sock]); + end1->fds_idx = pool_fds_add (pool, end1->sock, POOL_FD_READ); + if (end1->fds_idx < 0) abort(); + end1->events = POOL_FD_READ; pool->route_map[end1->sock] = end2; - if (end1->sock > pool->fd_max) pool->fd_max = end1->sock; - if (end1->sock < pool->fd_min) pool->fd_min = end1->sock; - FD_SET (end1->sock, &pool->fds_read); } // removing traces of connection end - reverse to what pool_set_conn_end() did static inline void -pool_reset_conn_end (pool_t* pool, int fd) +pool_reset_conn_end (pool_t* pool, pool_conn_end_t* end) { - int i; - - FD_CLR (fd, &pool->fds_read); - FD_CLR (fd, &pool->fds_write); - pool->route_map[fd] = NULL; - - if (fd == pool->fd_max) { - // fd_max can't be less than pool->ctl_recv, because of select() - for (i = fd - 1; i > pool->ctl_recv; i--) { - if (pool->route_map[i] != NULL) break; - } - pool->fd_max = i; - } - - if (fd == pool->fd_min) { - for (i = fd + 1; i < pool->fd_max; i++) { - if (pool->route_map[i] != NULL) break; - } - pool->fd_min = i; - } - - close (fd); + pool_fds_del (pool, end); + close (end->sock); + pool->route_map[end->sock] = NULL; } static void @@ -147,13 +275,14 @@ int dst_fd = dst->sock; pool_conn_end_t* src = pool->route_map[dst_fd]; - pool->n_conns--; - - if (glb_verbose) { - fprintf (stderr, "Pool %ld: disconnecting from %s " - "(total pool connections: %ld)\n", pool->id, - glb_socket_addr_to_string (&dst->dst_addr), pool->n_conns); +#ifndef NDEBUG + if (notify_router && !src->inc) { + glb_log_warn ("Connection close from server"); } +#endif + + pool->n_conns--; + pool->stats.conns_closed++; if (notify_router) glb_router_disconnect (pool->router, &dst->dst_addr); @@ -163,32 +292,34 @@ close (src->splice[0]); close (src->splice[1]); #endif + // in reverse order to pool_set_conn_end() in pool_handle_add_conn() + pool_reset_conn_end (pool, dst); + pool_reset_conn_end (pool, src); + if (dst->inc) { free (dst); // frees both ends } else { assert (src->inc); - free (src); + free (src); // frees both ends } - - pool_reset_conn_end (pool, src_fd); - pool_reset_conn_end (pool, dst_fd); } static void pool_handle_add_conn (pool_t* pool, pool_ctl_t* ctl) { pool_conn_end_t* inc_end = ctl->data; - pool_conn_end_t* dst_end = ctl->data + pool_conn_size/2; + pool_conn_end_t* dst_end = ctl->data + pool_end_size; pool_set_conn_end (pool, inc_end, dst_end); pool_set_conn_end (pool, dst_end, inc_end); pool->n_conns++; // increment connection count + pool->stats.conns_opened++; + if (glb_verbose) { - fprintf (stderr, - "Pool %ld: added connection, (total pool connections: %ld)\n", - pool->id, pool->n_conns); + glb_log_info ("Pool %ld: added connection, " + "(total pool connections: %ld)", pool->id, pool->n_conns); } } @@ -197,44 +328,96 @@ { const glb_sockaddr_t* dst = ctl->data; int fd; + int count = pool->fd_max - 1; // ctl_recv is not in route_map - for (fd = pool->fd_min; fd <= pool->fd_max; fd++) { + for (fd = 0; count; fd++) { pool_conn_end_t* end = pool->route_map[fd]; - if (end && glb_socket_addr_is_equal(&end->dst_addr, dst)) { - // remove conn, but don't try to notify router 'cause it's already - // dropped this destination - pool_remove_conn (pool, fd, false); + assert (fd < POOL_MAX_FD); + + if (end) { + count--; + if (glb_socket_addr_is_equal(&end->dst_addr, dst)) { + // remove conn, but don't try to notify router 'cause it's + // already dropped this destination + pool_remove_conn (pool, fd, false); + count--; // 1 connection means 2 file descriptors + } } } } +static inline void +pool_handle_stats (pool_t* pool, pool_ctl_t* ctl) +{ + glb_pool_stats_t* stats = ctl->data; + pool->stats.n_conns = pool->n_conns; + glb_pool_stats_add (stats, &pool->stats); + pool->stats = glb_zero_stats; +} + +#define GLB_MUTEX_LOCK(mtx) \ +{ \ + int ret; \ + if ((ret = pthread_mutex_lock (mtx))) { \ + glb_log_fatal ("Failed to lock mutex: %d (%s)", ret, strerror(ret));\ + abort(); \ + } \ +} + +#define GLB_MUTEX_UNLOCK(mtx) \ +{ \ + int ret; \ + if ((ret = pthread_mutex_unlock (mtx))) { \ + glb_log_fatal ("Failed to unlock mutex: %d (%s)", ret, strerror(ret));\ + abort(); \ + } \ +} + static long -pool_handle_ctl (pool_t* pool, pool_ctl_t* ctl) +pool_handle_ctl (pool_t* pool) { - switch (ctl->code) { + pool_ctl_t ctl; + register long ret; + + // remove ctls from poll count to get only traffic polls + pool->stats.n_polls--; + + ret = read (pool->ctl_recv, &ctl, sizeof(ctl)); + + if (sizeof(ctl) != ret) { // incomplete ctl read, should never happen + glb_log_fatal ("Incomplete read from ctl, errno: %d (%s)", + errno, strerror (errno)); + abort(); + } + + switch (ctl.code) { case POOL_CTL_ADD_CONN: - pool_handle_add_conn (pool, ctl); + pool_handle_add_conn (pool, &ctl); break; case POOL_CTL_DROP_DST: - pool_handle_drop_dst (pool, ctl); + pool_handle_drop_dst (pool, &ctl); + break; + case POOL_CTL_STATS: + pool_handle_stats (pool, &ctl); break; default: // nothing else is implemented - fprintf (stderr, "Unsupported CTL: %d\n", ctl->code); + glb_log_warn ("Unsupported CTL: %d\n", ctl.code); } // Notify ctl sender - pthread_mutex_lock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); pthread_cond_signal (&pool->cond); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return 0; } static inline ssize_t -pool_send_data (pool_t* pool, pool_conn_end_t* dst, bool reset_fds_read) +pool_send_data (pool_t* pool, pool_conn_end_t* dst, pool_conn_end_t* src) { - ssize_t ret; + ssize_t ret; + uint32_t dst_events = dst->events; #ifndef GLB_USE_SPLICE ret = send (dst->sock, &dst->buf[dst->sent], dst->total - dst->sent, @@ -245,46 +428,62 @@ #endif if (ret > 0) { -#ifdef GLB_POOL_STATS pool->stats.send_bytes += ret; -#endif + pool->stats.tx_bytes += dst->inc * ret; + dst->sent += ret; - if (dst->sent == dst->total) { // all data sent, reset pointers - dst->sent = dst->total = 0; - FD_CLR (dst->sock, &pool->fds_write); - } - else { -// perror ("Pool: incomplete send"); - FD_SET (dst->sock, &pool->fds_write); - } - if (reset_fds_read && (dst->total < pool_buf_size)) { - // some space exists in the buffer, - // reestablish src_fd in pool->fds_read - int src_sock = pool->route_map[dst->sock]->sock; - FD_SET (src_sock, &pool->fds_read); + if (dst->sent == dst->total) { // all data sent, reset pointers + dst->sent = dst->total = 0; + dst_events &= ~POOL_FD_WRITE; // clear WRITE flag + } + else { // there is unsent data left + glb_log_debug ("Setting WRITE flag on %s: sent = %zu, total = " + "%zu, bufsiz = %zu", dst->inc ? "client" : "server", + dst->sent, dst->total,pool_buf_size); + dst_events \|= POOL_FD_WRITE; // set WRITE flag + } + + if (src && !(src->events & POOL_FD_READ) && + (dst->total < pool_buf_size)) { + // some space exists in the buffer, reestablish READ flag in src + src->events \|= POOL_FD_READ; + pool_fds_set_events (pool, src); } } else { - switch (errno) { + ret = -errno; + switch (-ret) { case ESPIPE: case EBUSY: case EINTR: case ENOBUFS: case EAGAIN: - FD_SET (dst->sock, &pool->fds_write); + glb_log_debug ("Send data error: %d (%s)", -ret, strerror(-ret)); + dst_events \|= POOL_FD_WRITE; ret = 0; // pretend nothing happened break; case EPIPE: + glb_log_debug ("pool_remove_conn() from pool_send_data()"); pool_remove_conn(pool, dst->sock, true); break; default: - perror ("Pool: send failed"); - printf ("ALARM!!! missed case: errno = %d\n", errno); + glb_log_warn ("Send data failed, unhandled error: %d (%s)", + -ret, strerror(-ret)); } } -#ifdef GLB_POOL_STATS - pool->stats.n_send++; -#endif + + pool->stats.n_send++; + + if (dst_events != dst->events) { // events changed + glb_log_debug ("Old flags on %s: %s %s", dst->inc ? "client":"server", + dst->events & POOL_FD_READ ? "POOL_FD_READ":"", + dst->events & POOL_FD_WRITE ? "POOL_FD_WRITE":""); + dst->events = dst_events; + pool_fds_set_events (pool, dst); + glb_log_debug ("New flags on %s: %s %s", dst->inc ? "client":"server", + dst->events & POOL_FD_READ ? "POOL_FD_READ":"", + dst->events & POOL_FD_WRITE ? "POOL_FD_WRITE":""); + } return ret; } @@ -296,6 +495,8 @@ ssize_t ret = 0; pool_conn_end_t* dst = pool->route_map[src_fd]; +// glb_log_debug ("pool_handle_read()"); + // first, try read data from source, if there's enough space if (dst->total < pool_buf_size) { #ifndef GLB_USE_SPLICE @@ -308,33 +509,44 @@ #endif if (ret > 0) { dst->total += ret; - // now try to send whatever we have - // (since we're here, we're in fds_read, no need to reset) - if (pool_send_data (pool, dst, false) < 0) { - // probably don't care what error is - perror ("pool_handle_read(): sending data"); + // now try to send whatever we have received so far + // (since we're here, POOL_FD_READ on src is not cleared, no need + // to set it once again) + ssize_t send_err; + if ((send_err = pool_send_data (pool, dst, NULL)) < 0) { + glb_log_warn ("pool_send_data(): %zd (%s)", + -send_err, strerror(-send_err)); } + if (dst->total == pool_buf_size) { - // no space for next read, remove from fds_read - FD_CLR (src_fd, &pool->fds_read); + // no space for next read, clear POOL_FD_READ + pool_conn_end_t* src = pool->route_map[dst->sock]; + assert (src->events & POOL_FD_READ); + src->events &= ~POOL_FD_READ; + pool_fds_set_events (pool, src); } -#ifdef GLB_POOL_STATS + pool->stats.recv_bytes += ret; -#endif + + // increment only if it's coming from incoming interface + pool->stats.rx_bytes += (!dst->inc) * ret; } else { if (0 == ret) { // socket closed, must close another end and cleanup +// glb_log_debug ("pool_remove_conn() from pool_handle_read()"); pool_remove_conn (pool, src_fd, true); - ret = -1; + ret = -EPIPE; } else { // some other error - if (errno != EAGAIN) - perror ("pool_handle_read(): receiving data"); + if (errno != EAGAIN) { + ret = -errno; + glb_log_warn ("pool_handle_read(): %zd (%s)", + -ret, strerror(-ret)); + } } } -#ifdef GLB_POOL_STATS + pool->stats.n_recv++; -#endif } return ret; } @@ -342,117 +554,147 @@ static inline ssize_t pool_handle_write (pool_t* pool, int dst_fd) { - register int src_fd = pool->route_map[dst_fd]->sock; - pool_conn_end_t* dst = pool->route_map[src_fd]; + pool_conn_end_t* src = pool->route_map[dst_fd]; + pool_conn_end_t* dst = pool->route_map[src->sock]; + + glb_log_debug ("pool_handle_write() to %s: %zu", + dst->inc ? "client" : "server", dst->total - dst->sent); if (dst->total) { + ssize_t send_err; + assert (dst->total > dst->sent); - // if (dst->total == pool_buf_size), source was removed from fds_read - if (pool_send_data (pool, dst, (dst->total == pool_buf_size)) < 0) { - // probably don't care what error is - perror ("pool_handle_read(): sending data error"); + + if ((send_err = pool_send_data (pool, dst, src)) < 0) { + glb_log_warn ("pool_send_data(): %zd (%s)", + -send_err, strerror(-send_err)); } } return 0; } +// returns on error or after handling ctl - the latter may cause changes in +// file descriptors. +static inline long +pool_handle_events (pool_t* pool, long count) +{ + long idx; +#ifdef USE_EPOLL + for (idx = 0; idx < count; idx++) { + pollfd_t* pfd = pool->pollfds + idx; + if (pfd->events & POOL_FD_READ) { + + if (pfd->data.fd != pool->ctl_recv) { // normal read + register long ret; + pool->stats.poll_reads++; + + ret = pool_handle_read (pool, pfd->data.fd); + if (ret < 0) return ret; + } + else { // ctl read + return pool_handle_ctl (pool); + } + } + if (pfd->events & POOL_FD_WRITE) { + register long ret; + pool->stats.poll_writes++; + + assert (pfd->data.fd != pool->ctl_recv); + ret = pool_handle_write (pool, pfd->data.fd); + if (ret < 0) return ret; + } + } +#else // POLL + if (pool->pollfds[0].revents & POOL_FD_READ) { // first, check ctl socket + return pool_handle_ctl (pool); + } + + for (idx = 1; count > 0; idx++) + { + pollfd_t* pfd = pool->pollfds + idx; + + assert (idx < pool->fd_max); + + if (pfd->revents) { + // revents might be less than pfd->revents because some of the + // pfd->events might be cleared in the previous loop + register ulong revents = pfd->revents & pfd->events; + + if (revents & POOL_FD_READ) { + register long ret; + pool->stats.poll_reads++; + + ret = pool_handle_read (pool, pfd->fd); + if (ret < 0) return ret; + } + if (revents & POOL_FD_WRITE) { + register long ret; + pool->stats.poll_writes++; + + ret = pool_handle_write (pool, pfd->fd); + if (ret < 0) return ret; + } + count--; + } + } +#endif // POLL + return 0; +} + static void* pool_thread (void* arg) { pool_t* pool = arg; // synchronize with the calling process - pthread_mutex_lock (&pool->lock); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); while (1) { long ret; - fd_set fds_read, fds_write; - fds_read = pool->fds_read; - fds_write = pool->fds_write; - ret = select (pool->fd_max+1, &fds_read, &fds_write, NULL, NULL); - - if (ret > 0) { // we have some input - long count = ret; - int fd = pool->fd_min; - // first, check ctl pipe - if (FD_ISSET (pool->ctl_recv, &fds_read)) { - pool_ctl_t ctl; - - ret = read (pool->ctl_recv, &ctl, sizeof(ctl)); - if (sizeof(ctl) == ret) { // complete ctl read - pool_handle_ctl (pool, &ctl); - } - else { // should never happen! - perror ("Pool: incomplete read from ctl"); - abort(); - } -#ifdef GLB_POOL_STATS - pool->stats.sel_reads++; -#endif - /* - * pool->ctl_recv can theoretically get between fd_min and - * fd_max. - * For simplicity we want to assume that it is not set when - * processing normal fds - * also set of connections could be modified by ctl, - * so start over - / - goto end; - } + ret = pool_fds_wait (pool); - assert (!FD_ISSET (pool->ctl_recv, &fds_read)); + if (ret > 0) { - // check remaining connections - while (count) { - assert (fd <= pool->fd_max); - - while (NULL == pool->route_map[fd]) fd++; - - / - * If pool_handle_read() or pool_handle_write() below - * return error, this is most likely because connection was - * closed. In that case cleanup has happend and fd set has - * changed. Break out of the loop to start over again. - / - if (FD_ISSET (fd, &fds_read)) { -#ifdef GLB_POOL_STATS - pool->stats.sel_reads++; -#endif - if (pool_handle_read (pool, fd) < 0) goto end; - count--; - } + pool->stats.n_polls++; - if (FD_ISSET (fd, &fds_write)) { -#ifdef GLB_POOL_STATS - pool->stats.sel_writes++; -#endif - if (pool_handle_write (pool, fd) < 0) goto end; - count--; - } + pool_handle_events (pool, ret); - fd++; - } } - else if (-1 == ret) { - perror ("select() failed"); + else if (ret < 0) { + glb_log_error ("pool_fds_wait() failed: %d (%s)", + errno, strerror(errno)); } else { - perror ("select() returned 0!"); - // timed out - //printf ("Thread %ld is idle\n", pool->id); + glb_log_error ("pool_fds_wait() interrupted: %d (%s)", + errno, strerror(errno)); } - end: -#ifdef GLB_POOL_STATS - pool->stats.n_select++; -#endif - continue; } return NULL; } +// initialize file descriptor set with ctl_recv descriptor +static long +pool_fds_init (pool_t pool, int ctl_fd) +{ + pool->fd_max = 0; + +#ifdef USE_EPOLL + pool->epoll_fd = epoll_create(FD_SETSIZE); + if (pool->epoll_fd < 0) { + glb_log_fatal ("epoll_create(%d) failed: %d (%s)", + FD_SETSIZE, errno, strerror(errno)); + return -errno; + } +#endif + + pool->pollfds_len = 0; + + return pool_fds_add (pool, ctl_fd, POOL_FD_READ); +} + static long pool_init (pool_t* pool, long id, glb_router_t* router) { @@ -461,32 +703,35 @@ pool->id = id; pool->router = router; + pool->stats = glb_zero_stats; pthread_mutex_init (&pool->lock, NULL); pthread_cond_init (&pool->cond, NULL); ret = pipe(pipe_fds); if (ret) { - perror ("Failed to open control pipe"); + ret = errno; + glb_log_fatal ("Failed to open control pipe: %d (%s)", + ret, strerror(ret)); return -ret; } pool->ctl_recv = pipe_fds[0]; pool->ctl_send = pipe_fds[1]; - FD_ZERO (&pool->fds_read); - FD_ZERO (&pool->fds_write); - FD_SET (pool->ctl_recv, &pool->fds_read); - pool->fd_max = pool->ctl_recv; - pool->fd_min = pool->fd_max; + ret = pool_fds_init (pool, pool->ctl_recv); + if (ret < 0) { + return ret; + } - // this, together with pthread_mutex_lock() in the beginning of + // this, together with GLB_MUTEX_LOCK() in the beginning of // pool_thread() avoids possible race in access to pool->thread - pthread_mutex_lock (&pool->lock); + GLB_MUTEX_LOCK (&pool->lock); ret = pthread_create (&pool->thread, NULL, pool_thread, pool); - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); if (ret) { - perror ("Failed to create thread"); + glb_log_fatal ("Failed to create pool thread: %d (%s)", + ret, strerror(ret)); return -ret; } @@ -509,17 +754,19 @@ for (i = 0; i < n_pools; i++) { if ((err = pool_init(&ret->pool[i], i, router))) { - fprintf (stderr, "Failed to initialize pool %ld\n", i); + glb_log_fatal ("Failed to initialize pool %ld.", i); abort(); } } } else { - fprintf (stderr, "Could not allocate memory for %zu pools\n", n_pools); + glb_log_fatal ("Could not allocate memory for %zu pools.", + n_pools); abort(); } - gettimeofday (&ret->begin, NULL); + ret->last_info = glb_time_now(); + ret->last_stats = ret->last_info; return ret; } @@ -533,7 +780,7 @@ pthread_join (p->thread, NULL); } // TODO: proper resource deallocation - fprintf (stderr, "glb_pool_destroy() not implemented yet!"); + glb_log_warn ("glb_pool_destroy() not implemented yet!"); } // finds the least busy pool @@ -554,25 +801,25 @@ } // Sends ctl and waits for confirmation from the pool thread -extern ssize_t +static ssize_t pool_send_ctl (pool_t* p, pool_ctl_t* ctl) { ssize_t ret; - pthread_mutex_lock (&p->lock); + GLB_MUTEX_LOCK (&p->lock); ret = write (p->ctl_send, ctl, sizeof (ctl)); if (ret != sizeof (ctl)) { - perror ("Sending ctl failed"); + glb_log_error ("Sending ctl failed: %d (%s)", errno, strerror(errno)); if (ret > 0) abort(); // partial ctl was sent, don't know what to do } else ret = 0; pthread_cond_wait (&p->cond, &p->lock); - pthread_mutex_unlock (&p->lock); + GLB_MUTEX_UNLOCK (&p->lock); return ret; } -extern long +long glb_pool_add_conn (glb_pool_t* pool, int inc_sock, int dst_sock, @@ -582,15 +829,12 @@ long ret = -ENOMEM; void* route = NULL; - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_add_conn(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); route = malloc (pool_conn_size); if (route) { pool_conn_end_t* inc_end = route; - pool_conn_end_t* dst_end = route + pool_conn_size / 2; + pool_conn_end_t* dst_end = route + pool_end_size; pool_ctl_t add_conn_ctl = { POOL_CTL_ADD_CONN, route }; inc_end->inc = true; @@ -612,39 +856,75 @@ ret = pool_send_ctl (p, &add_conn_ctl); } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return ret; } -long -glb_pool_drop_dst (glb_pool_t* pool, const glb_sockaddr_t* dst) +// Sends the same ctl to all pools. Returns 0 minus how many ctls failed +static inline long +pool_bcast_ctl (glb_pool_t* pool, pool_ctl_t* ctl) { - pool_ctl_t drop_dst_ctl = { POOL_CTL_DROP_DST, (void)dst }; ulong i; long ret = 0; - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_add_conn(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); for (i = 0; i < pool->n_pools; i++) { - ret -= (pool_send_ctl (&pool->pool[i], &drop_dst_ctl) < 0); + ret -= (pool_send_ctl (&pool->pool[i], ctl) < 0); } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); return ret; } -size_t +long +glb_pool_drop_dst (glb_pool_t pool, const glb_sockaddr_t* dst) +{ + pool_ctl_t drop_dst_ctl = { POOL_CTL_DROP_DST, (void)dst }; + return pool_bcast_ctl (pool, &drop_dst_ctl); +} + +ssize_t glb_pool_print_stats (glb_pool_t pool, char* buf, size_t buf_len) { - size_t len = 0; - long i; - struct timeval now; - double seconds; + glb_pool_stats_t stats = glb_zero_stats; + pool_ctl_t stats_ctl = { POOL_CTL_STATS, (void)&stats }; + ssize_t ret; + glb_time_t now = glb_time_now(); + + ret = pool_bcast_ctl (pool, &stats_ctl); + if (!ret) { + double elapsed = now - pool->last_stats; + ret = snprintf (buf, buf_len, "in: %lu out: %lu " + "recv: %lu / %lu send: %lu / %lu " + "conns: %lu / %lu poll: %lu / %lu / %lu " + "elapsed: %.5f\n", + stats.rx_bytes, stats.tx_bytes, + stats.recv_bytes, stats.n_recv, + stats.send_bytes, stats.n_send, + stats.conns_opened, stats.n_conns, + stats.poll_reads, stats.poll_writes, stats.n_polls, + elapsed); + } + else { + assert (ret < 0); + glb_log_error ("Failed to get stats from %d thread pools.", ret); + } + + pool->last_stats = now; + + return ret; +} + +ssize_t +glb_pool_print_info (glb_pool_t pool, char* buf, size_t buf_len) +{ + size_t len = 0; + long i; + glb_time_t now; + double elapsed; #ifndef GLB_POOL_STATS len += snprintf (buf + len, buf_len - len, "Pool: connections per thread:"); @@ -654,31 +934,28 @@ } #endif - if (pthread_mutex_lock (&pool->lock)) { - perror ("glb_pool_print_stats(): failed to lock mutex"); - abort(); - } + GLB_MUTEX_LOCK (&pool->lock); - gettimeofday (&now, NULL); - seconds = now.tv_sec - pool->begin.tv_sec + - (now.tv_usec - pool->begin.tv_usec) * 1.0e-06; + now = glb_time_now (); + elapsed = now - pool->last_info; for (i = 0; i < pool->n_pools; i++) { #ifdef GLB_POOL_STATS pool_stats_t s = pool->pool[i].stats; pool->pool[i].stats = zero_stats; + len += snprintf (buf + len, buf_len - len, "Pool %2ld: conns: %5ld, selects: %9zu (%9.2f sel/sec)\n" "recv : %9zuB %9zuR %9zuS %9.2fB/R %9.2fB/sec %9.2fR/S %9.2fR/sec\n" "send : %9zuB %9zuW %9zuS %9.2fB/W %9.2fB/sec %9.2fW/S %9.2fW/sec\n", - i, pool->pool[i].n_conns, s.n_select, (double)s.n_select/seconds, - s.recv_bytes,s.n_recv,s.sel_reads,(double)s.recv_bytes/s.n_recv, - (double)s.recv_bytes/seconds,(double)s.n_recv/s.n_select, - (double)s.n_recv/seconds, - s.send_bytes,s.n_send,s.sel_writes,(double)s.send_bytes/s.n_send, - (double)s.send_bytes/seconds,(double)s.n_send/s.n_select, - (double)s.n_send/seconds + i, pool->pool[i].n_conns, s.n_polls, (double)s.n_polls/elapsed, + s.recv_bytes,s.n_recv,s.poll_reads,(double)s.recv_bytes/s.n_recv, + (double)s.recv_bytes/elapsed,(double)s.n_recv/s.n_polls, + (double)s.n_recv/elapsed, + s.send_bytes,s.n_send,s.poll_writes,(double)s.send_bytes/s.n_send, + (double)s.send_bytes/elapsed,(double)s.n_send/s.n_polls, + (double)s.n_send/elapsed ); if (len == buf_len) { buf[len - 1] = '\0'; @@ -694,7 +971,7 @@ #endif } - pthread_mutex_unlock (&pool->lock); + GLB_MUTEX_UNLOCK (&pool->lock); len += snprintf (buf + len, buf_len - len,"\n"); if (len == buf_len) { @@ -702,7 +979,7 @@ return (len - 1); } - pool->begin = now; + pool->last_info = now; return len; }
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_pool.h ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_pool.h 23 2008-06-22 14:21:39Z alex $ + * $Id: glb_pool.h 59 2009-04-16 10:30:17Z alex $ / #ifndef _glb_pool_h_ @@ -9,6 +9,7 @@ #include "glb_socket.h" #include "glb_router.h" +#include "glb_pool_stats.h" typedef struct glb_pool glb_pool_t; @@ -30,7 +31,10 @@ extern long glb_pool_drop_dst (glb_pool_t pool, const glb_sockaddr_t* dst); -extern size_t +extern ssize_t glb_pool_print_stats (glb_pool_t* pool, char* buf, size_t buf_len); +extern ssize_t +glb_pool_print_info (glb_pool_t* pool, char* buf, size_t buf_len); + #endif // _glb_pool_h_
[-] [+]	Added	glb-0.7.2.tar.bz2/src/glb_pool_stats.h ^
@@ -0,0 +1,46 @@ +/* + * Copyright (C) 2008 Codership Oy <info@codership.com> + * + * $Id: glb_pool_stats.h 59 2009-04-16 10:30:17Z alex $ + / + +#ifndef _glb_pool_stats_h_ +#define _glb_pool_stats_h_ + +typedef struct glb_pool_stats +{ + ulong rx_bytes; // bytes received on incoming interface (from clients) + ulong tx_bytes; // bytes transmittted from incoming interface + ulong recv_bytes; // bytes received by recv() calls + ulong n_recv; // number of recv() calls + ulong send_bytes; // bytes sent by send() calls + ulong n_send; // number of send() calls + ulong conns_opened; // number of opened connections + ulong conns_closed; // number of closed connections + ulong n_conns; // number of current connections + ulong poll_reads; // number of read-ready fd's returned by poll() + ulong poll_writes; // number of write-ready fd's returned by poll() + ulong n_polls; // number of poll() calls +} glb_pool_stats_t; + +static const glb_pool_stats_t glb_zero_stats = { 0, }; + +// adds right stats to left stats +static inline void +glb_pool_stats_add (glb_pool_stats_t left, glb_pool_stats_t* right) +{ + left->rx_bytes += right->rx_bytes; + left->tx_bytes += right->tx_bytes; + left->recv_bytes += right->recv_bytes; + left->n_recv += right->n_recv; + left->send_bytes += right->send_bytes; + left->n_send += right->n_send; + left->conns_opened += right->conns_opened; + left->conns_closed += right->conns_closed; + left->n_conns += right->n_conns; + left->poll_reads += right->poll_reads; + left->poll_writes += right->poll_writes; + left->n_polls += right->n_polls; +} + +#endif // _glb_pool_stats_h_
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_router.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_router.c 38 2008-07-15 18:03:19Z alex $ + * $Id: glb_router.c 59 2009-04-16 10:30:17Z alex $ / #include <pthread.h> @@ -10,6 +10,7 @@ #include <errno.h> #include <stdio.h> #include <unistd.h> // for close() +#include <time.h> #include "glb_log.h" #include "glb_socket.h" @@ -20,10 +21,9 @@ typedef struct router_dst { glb_dst_t dst; - double weight; - long conns; // how many connections use this destination - double usage; // usage measure: weight/(conns + 1) - bigger wins - bool ready; // if destinaiton accepts connecitons + long conns; // how many connections use this destination + double usage; // usage measure: weight/(conns + 1) - bigger wins + time_t failed; // last time connection to this destination failed } router_dst_t; struct glb_router @@ -34,6 +34,12 @@ router_dst_t dst; }; +static const double router_div_prot = 1.0e-09; +static inline double +router_dst_usage (router_dst_t* d) +{ return (d->dst.weight / (d->conns + router_div_prot)); } + + long glb_router_change_dst (glb_router_t* router, const glb_dst_t* dst) { @@ -71,8 +77,8 @@ else if (d->dst.weight != dst->weight) { // update weight and usage d->dst.weight = dst->weight; - d->weight = dst->weight; - d->usage = d->weight / (d->conns + 1); +// d->weight = dst->weight; + d->usage = router_dst_usage (d); } goto out; } @@ -98,10 +104,10 @@ d = router->dst + router->n_dst; router->n_dst++; d->dst = dst; - d->weight = dst->weight; +// d->weight = dst->weight; d->conns = 0; - d->usage = d->weight / (d->conns + 1); - d->ready = true; + d->usage = router_dst_usage(d); + d->failed = 0; out: assert (router->n_dst >= 0); @@ -149,6 +155,9 @@ router_cleanup (router); } +// seconds (should be > 1 due to time_t precision) +static const double DST_RETRY_INTERVAL = 2.0; + // find a ready destination with minimal usage static router_dst_t router_choose_dst (glb_router_t* router) @@ -158,10 +167,13 @@ if (router->n_dst > 0) { double max_usage = 0.0; int i; + time_t now = time(NULL); for (i = 0; i < router->n_dst; i++) { router_dst_t* d = &router->dst[i]; - if (d->ready && d->usage > max_usage) { + + if (d->usage > max_usage && + difftime (now, d->failed) > DST_RETRY_INTERVAL) { ret = d; max_usage = d->usage; } @@ -176,37 +188,28 @@ router_connect_dst (glb_router_t* router, int sock, glb_sockaddr_t* addr) { router_dst_t* dst; + int error = 1; + // keep trying until we run out of destinations while ((dst = router_choose_dst (router))) { if (connect (sock, (struct sockaddr)&dst->dst.addr, sizeof (dst->dst.addr))) { - // connect failed, mark destination bad - fprintf (stderr, "Router: failed to connect to %s\n", - glb_socket_addr_to_string (&dst->dst.addr)); - dst->ready = false; - return -1; + error = errno; + // connect failed, update destination failed mark + fprintf (stderr, "Router: failed to connect to %s: %s\n", + glb_socket_addr_to_string (&dst->dst.addr), strerror(error)); + dst->failed = time(NULL); } else { // success, update stats dst->conns++; - dst->usage = dst->weight / (dst->conns + 1); + dst->usage = router_dst_usage(dst); addr = dst->dst.addr; return 0; } } - return -1; -} - -// reset ready flag on destinations -// (TODO: associate a timestamp with a flag, don't reset right away) -static void -router_reset_dst (glb_router_t* router) -{ - long i; - for (i = 0; i < router->n_dst; i++) { - router->dst[i].ready = true; - } + return -error; // all attempts failed, return last errno } // returns error code @@ -219,7 +222,7 @@ sock = glb_socket_create (&router->sock_out); if (sock < 0) { perror ("Router: glb_socket_create"); - goto out; + return sock; } if (pthread_mutex_lock (&router->lock)) { @@ -229,7 +232,6 @@ // attmept to connect until we run out of destinations ret = router_connect_dst (router, sock, dst_addr); - router_reset_dst(router); // avoid socket leak if (ret < 0) { @@ -238,8 +240,8 @@ sock = ret; } -out: pthread_mutex_unlock (&router->lock); + return sock; } @@ -257,7 +259,7 @@ router_dst_t* d = &router->dst[i]; if (glb_socket_addr_is_equal (&d->dst.addr, dst)) { d->conns--; - d->usage = d->weight / (d->conns + 1); + d->usage = router_dst_usage(d); break; } } @@ -271,7 +273,7 @@ } size_t -glb_router_print_stats (glb_router_t* router, char* buf, size_t buf_len) +glb_router_print_info (glb_router_t* router, char* buf, size_t buf_len) { size_t len = 0; long total_conns = 0; @@ -296,9 +298,10 @@ total_conns += d->conns; - len += snprintf (buf + len, buf_len - len, "%s : %6ld %9.3f %6ld\n", + len += snprintf (buf + len, buf_len - len, "%s : %8.3f %7.3f %5ld\n", glb_socket_addr_to_string(&d->dst.addr), - d->dst.weight, 1.0 - (d->usage/d->weight), d->conns); + d->dst.weight, 1.0/(d->usage + 1.0), + d->conns); if (len == buf_len) { buf[len - 1] = '\0'; return (len - 1);
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_router.h ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_router.h 35 2008-07-14 23:54:34Z alex $ + * $Id: glb_router.h 59 2009-04-16 10:30:17Z alex $ / #ifndef _glb_router_h_ @@ -42,6 +42,6 @@ // Returns the length of the string extern size_t -glb_router_print_stats (glb_router_t router, char* buf, size_t buf_len); +glb_router_print_info (glb_router_t* router, char* buf, size_t buf_len); #endif // _glb_router_h_
[-] [+]	Changed	glb-0.7.2.tar.bz2/src/glb_socket.c ^
@@ -1,7 +1,7 @@ /* * Copyright (C) 2008 Codership Oy <info@codership.com> * - * $Id: glb_socket.c 31 2008-06-30 17:17:14Z alex $ + * $Id: glb_socket.c 65 2009-06-06 21:16:51Z alex $ / #include <stdio.h> @@ -12,9 +12,11 @@ #include <assert.h> #include <unistd.h> +#include "glb_log.h" #include "glb_socket.h" -//static const size_t addr_string_len = 512; heh, my GCC refuses to see it as a constant! here goes type safety... +//static const size_t addr_string_len = 512; heh, my GCC refuses to see it as +//a constant! here goes type safety... #define addr_string_len 512 static char addr_string[addr_string_len] = { 0, }; @@ -87,7 +89,8 @@ sock = socket (PF_INET, SOCK_STREAM, 0); if (sock < 0) { - perror ("glb_socket_create(): socket"); + glb_log_error ("Failed to create listening socket: %d (%s)", + errno, strerror (errno)); return -errno; } #if 0 @@ -107,7 +110,8 @@ / Give the socket a name. / if (bind (sock, (struct sockaddr ) addr, sizeof (*addr)) < 0) { - perror ("glb_socket_create(): bind"); + glb_log_error ("Failed to bind listening socket: %d (%s)", + errno, strerror (errno)); close (sock); return -errno; }
[-] [+]	Added	glb-0.7.2.tar.bz2/src/glb_time.h ^
@@ -0,0 +1,22 @@ +/* + * Copyright (C) 2008 Codership Oy <info@codership.com> + * + * $Id: glb_time.h 57 2009-04-15 17:36:21Z alex $ + / + +#ifndef _glb_time_h_ +#define _glb_time_h_ + +#include <sys/time.h> +#include <time.h> + +typedef double glb_time_t; + +static inline glb_time_t +glb_time_now() { + struct timeval tv; + gettimeofday (&tv, NULL); + return (1.0e-06 tv.tv_usec + tv.tv_sec); +} + +#endif // _glb_time_h_

Changes of Revision 7