Lighttpd

Under heavy concurrent large file downloads (400 connections, 48MB/sec), using Strace shows lots of sendfile64 resource unavailable. Viewing the server-counters says that 75% are sync and only 25% async. At the same time iowait goes from very low (0-5%) up to 45%

Seems like we are hitting a bottleneck somewhere, not sure if it is lighttpd or the OS. Any ideas?
Let me know if any additional information is needed. Below is a sampling of the strace output...

sendfile64(935, 1079, [3500833], 2097152) = -1 EAGAIN (Resource temporarily unavailable)
epoll_ctl(6, EPOLL_CTL_ADD, 935, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=935, u64=935}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 1289, {0, {u32=0, u64=0}}) = 0
open("/dev/shm/l-RT79d2", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 764
unlink("/dev/shm/l-RT79d2")             = 0
ftruncate64(764, 446464)                = 0
mmap2(NULL, 446464, PROT_READ|PROT_WRITE, MAP_SHARED, 764, 0) = 0xb56cf000
munmap(0xb56cf000, 446464)              = 0
close(764)                              = 0
sendfile64(1289, 1333, [27326661], 450485) = 159720
open("/dev/shm/l-kclyJl", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 764
unlink("/dev/shm/l-kclyJl")             = 0
ftruncate64(764, 286720)                = 0
mmap2(NULL, 286720, PROT_READ|PROT_WRITE, MAP_SHARED, 764, 0) = 0xb56f6000
munmap(0xb56f6000, 286720)              = 0
close(764)                              = 0
sendfile64(1289, 1333, [27486381], 290765) = -1 EAGAIN (Resource temporarily unavailable)
epoll_ctl(6, EPOLL_CTL_ADD, 1289, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=1289, u64=1289}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 608, {0, {u32=0, u64=0}}) = 0
sendfile64(608, 192, [1965160], 131992) = 131992
munmap(0xa3cde000, 2097152)             = 0
close(192)                              = 0
open("/dev/shm/l-LFy5eF", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 192
unlink("/dev/shm/l-LFy5eF")             = 0
ftruncate64(192, 2097152)               = 0
mmap2(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_SHARED, 192, 0) = 0xa3cde000
io_submit(3080347648, 1, {...})         = 1
epoll_ctl(6, EPOLL_CTL_DEL, 686, {0, {u32=0, u64=0}}) = 0
sendfile64(686, 74, [699688], 1397464)  = 72600
epoll_ctl(6, EPOLL_CTL_ADD, 686, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=686, u64=686}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 446, {0, {u32=0, u64=0}}) = 0
sendfile64(446, 527, [591300], 1505852) = 42340
epoll_ctl(6, EPOLL_CTL_ADD, 446, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=446, u64=446}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 913, {0, {u32=0, u64=0}}) = 0
sendfile64(913, 884, [102200], 1994952) = 56940
epoll_ctl(6, EPOLL_CTL_ADD, 913, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=913, u64=913}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 871, {0, {u32=0, u64=0}}) = 0
open("/dev/shm/l-4WN8KY", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 764
unlink("/dev/shm/l-4WN8KY")             = 0
ftruncate64(764, 1105920)               = 0
mmap2(NULL, 1105920, PROT_READ|PROT_WRITE, MAP_SHARED, 764, 0) = 0xb562e000
munmap(0xb562e000, 1105920)             = 0
close(764)                              = 0
sendfile64(871, 1207, [29450208], 1108258) = 68340
open("/dev/shm/l-jYdihi", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 764
unlink("/dev/shm/l-jYdihi")             = 0
ftruncate64(764, 1036288)               = 0
mmap2(NULL, 1036288, PROT_READ|PROT_WRITE, MAP_SHARED, 764, 0) = 0xb563f000
munmap(0xb563f000, 1036288)             = 0
close(764)                              = 0
sendfile64(871, 1207, [29518548], 1039918) = -1 EAGAIN (Resource temporarily unavailable)
epoll_ctl(6, EPOLL_CTL_ADD, 871, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=871, u64=871}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 1039, {0, {u32=0, u64=0}}) = 0
sendfile64(1039, 122, [1633500], 463652) = 58080
epoll_ctl(6, EPOLL_CTL_ADD, 1039, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=1039, u64=1039}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 481, {0, {u32=0, u64=0}}) = 0
sendfile64(481, 14, [685868], 1411284)  = 91476
epoll_ctl(6, EPOLL_CTL_ADD, 481, {EPOLLOUT|EPOLLERR|EPOLLHUP, {u32=481, u64=481}}) = 0
epoll_ctl(6, EPOLL_CTL_DEL, 167, {0, {u32=0, u64=0}}) = 0
open("/dev/shm/l-AMoENB", O_RDWR|O_CREAT|O_EXCL|O_LARGEFILE, 0600) = 764
unlink("/dev/shm/l-AMoENB")             = 0
ftruncate64(764, 2097152)               = 0
mmap2(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_SHARED, 764, 0) = 0x97a00000