TFO client

Hello,

I'm trying to use TCP Fast Open with unbound 1.10.1. The server side of
TFO seems to be working as expected as I see the setsockopt() options
for the IPv4, IPv6 and control sockets:

root@vm-fu1:~# strace -s 1024 -o /tmp/unbound /usr/sbin/unbound -d -p
root@vm-fu1:~grep -F FASTOPEN /tmp/unbound
setsockopt(4, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(5, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(6, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0

TFO on the client side works when using plain TCP:

root@vm-fu1:~# strace -s 1024 -o /tmp/tcp-plain /usr/sbin/unbound -d -p
root@vm-fu1:~# grep -F FASTOPEN /tmp/tcp-plain
setsockopt(4, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(5, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(6, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
sendmsg(14, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16,
msg_iov=[{iov_base="\0'", iov_len=2},
{iov_base="\345;\1\0\0\1\0\0\0\0\0\1\6google\3com\0\0\2\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=39}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 41
sendmsg(15, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16,
msg_iov=[{iov_base="\0\34", iov_len=2},
{iov_base="\306,\1\20\0\1\0\0\0\0\0\1\0\0000\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=28}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 30
sendmsg(16, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16,
msg_iov=[{iov_base="\0%", iov_len=2},
{iov_base="\235\n\1\0\0\1\0\0\0\0\0\1\10_ta-4f66\0\0\n\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=37}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 39
sendmsg(14, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16, msg_iov=[{iov_base="\0
", iov_len=2},
{iov_base="\5!\1\20\0\1\0\0\0\0\0\1\3com\0\0+\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=32}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 34
sendmsg(15, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16, msg_iov=[{iov_base="\0
", iov_len=2},
{iov_base=".\276\1\20\0\1\0\0\0\0\0\1\3com\0\0000\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=32}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 34
sendmsg(14, {msg_name={sa_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("8.8.8.8")}, msg_namelen=16,
msg_iov=[{iov_base="\0'", iov_len=2},
{iov_base="\34\v\1\20\0\1\0\0\0\0\0\1\6google\3com\0\0+\0\1\0\0)\20\0\0\0\200\0\0\0",
iov_len=39}], msg_iovlen=2, msg_controllen=0, msg_flags=0},
MSG_FASTOPEN) = 41

but doesn't when using DoT:

root@vm-fu1:~# strace -s 1024 -o /tmp/dot /usr/sbin/unbound -d -p
root@vm-fu1:~# grep -F FASTOPEN /tmp/dot
setsockopt(4, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(5, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0
setsockopt(6, SOL_TCP, TCP_FASTOPEN, [5], 4) = 0

Here is the config I'm using in which I toggled the last 3 lines for
plain TCP and DoT:

server:
  verbosity: 4
  extended-statistics: yes
  interface: 0.0.0.0
  access-control: 0.0.0.0/0 allow
  tls-cert-bundle: /etc/ssl/certs/ca-certificates.crt
  tcp-upstream: yes
forward-zone:
  name: .
  #forward-addr: 8.8.8.8
  forward-addr: 8.8.8.8@853#dns.google
  forward-tls-upstream: yes

root@vm-fu1:~# unbound -V
Version 1.10.1

Configure line: --build=x86_64-linux-gnu --prefix=/usr
--includedir=${prefix}/include --mandir=${prefix}/share/man
--infodir=${prefix}/share/info --sysconfdir=/etc --localstatedir=/var
--disable-silent-rules --libdir=${prefix}/lib/x86_64-linux-gnu
--libexecdir=${prefix}/lib/x86_64-linux-gnu --disable-maintainer-mode
--disable-dependency-tracking --disable-rpath
--with-pidfile=/run/unbound.pid
--with-rootkey-file=/var/lib/unbound/root.key --with-libevent
--with-pythonmodule --enable-subnet --enable-tfo-client
--enable-tfo-server --enable-systemd --with-chroot-dir= --libdir=/usr/lib
Linked libs: libevent 2.1.11-stable (it uses epoll), OpenSSL 1.1.1f 31
Mar 2020
Linked modules: dns64 python subnetcache respip validator iterator
TCP Fastopen feature available

BSD licensed, see LICENSE in source package for details.
Report bugs to unbound-bugs@nlnetlabs.nl or
https://github.com/NLnetLabs/unbound/issues

root@vm-fu1:~# uname -a
Linux vm-fu1 5.4.0-40-generic #44-Ubuntu SMP Tue Jun 23 00:01:04 UTC
2020 x86_64 x86_64 x86_64 GNU/Linux

root@vm-fu1:~# sysctl net.ipv4.tcp_fastopen
net.ipv4.tcp_fastopen = 3

Am I doing something wrong or is this the expected behaviour? or a bug?

Regards,
Simon

Hi Simon,

The reason the TFO client does not work for DoT, is because openssl does
not support it. Unbound needs support in openssl to have client side
TCP fast open for TLS connections.

The fastopen wants the first sendmsg call with data to have the
MSG_FASTOPEN option. With openssl, openssl wants to perform that first
data write as part of the handshake. If it does not set that
MSG_FASTOPEN, Unbound cannot really do anything about it. Hence unbound
falls back to using a plain connect, so that it works.

Best regards, Wouter

Hi Wouter,

The reason the TFO client does not work for DoT, is because openssl does
not support it. Unbound needs support in openssl to have client side
TCP fast open for TLS connections.

The fastopen wants the first sendmsg call with data to have the
MSG_FASTOPEN option. With openssl, openssl wants to perform that first
data write as part of the handshake. If it does not set that
MSG_FASTOPEN, Unbound cannot really do anything about it. Hence unbound
falls back to using a plain connect, so that it works.

Thanks for the explanation. Looking at OpenSSL, I found [1] and [2] that
_seem_ to indicate that 1.1.1 and later do support TFO, if I'm
understanding right. Maybe there is a way to tell OpenSSL to use TFO for
the DoT client connections too?

1: https://github.com/openssl/openssl/issues/4783
2: https://github.com/openssl/openssl/pull/4802

Regards,
Simon