All the examples I have found so far either only read OR write or were 10000 line beasts where I didn't even know where to start to understand how they work.
To test my code I pointed a browser at my server and sent a simple http request. The results are confusing.
For example at one point GetQueuedCompletionStatus returns and WSARecv says it read the number of bytes of the http response I sent although this response should (and does) end up at the client and the recvbuffer isn't even filled with those bytes.
Also I don't understand when to free my buffers once the other browser closes the connection since GetQueuedCompletionStatus keeps returning a few times after my call to closesocket.
Further I don't know when there is data to read or data to write once GetQueuedCompletionStatus returns. I could just try both and see which fails but that seems rude.
To reveal any misconceptions I might have about IOCP I wrote some pseudo code to convey what I think my code does:
main {
create server socket
create io completion port
while true {
accept client socket
create completion port for client socket
create recv buffer and send buffer for client
call WSARecv once with 0 bytes for whatever reason
}
}
worker thread {
while true {
wait until GetQueuedCompletionStatus returns
do something if that failed, not quite sure what (free buffers?)
if no bytes were transferred, close socket
try to recv data
try to send data
}
}
Actual code:
#include <stdio.h>
#include <stdlib.h>
#include <winsock2.h>
#define BUFFER_SIZE 1024
typedef struct {
WSAOVERLAPPED overlapped;
SOCKET socket;
WSABUF sendbuf;
WSABUF recvbuf;
char sendbuffer[BUFFER_SIZE];
char recvbuffer[BUFFER_SIZE];
} client;
DWORD WINAPI worker_thread(HANDLE iocp){
DWORD flags = 0, n = 0;
ULONG unused;
client *c;
while (1){
int ret = GetQueuedCompletionStatus(iocp, &n, &unused, (LPOVERLAPPED*)&c, INFINITE);
printf("%3d triggered\n", c->socket);
if (ret == FALSE){
printf("%3d GetQueuedCompletionStatus error %i\n", c->socket, WSAGetLastError());
continue;
}
if (c->socket == INVALID_SOCKET){
printf("error: socket already closed\n");
continue;
}
if (n == 0) {
printf("%3d disconnected\n", c->socket);
closesocket(c->socket);
c->socket = INVALID_SOCKET;
continue;
}
/* how do I know if there is data to read or data to write? */
WSARecv(c->socket, &(c->recvbuf), 1, &n, &flags, &(c->overlapped), NULL);
printf("%3d WSARecv %ld bytes\n", c->socket, n);
WSASend(c->socket, &(c->sendbuf), 1, &n, flags, &(c->overlapped), NULL);
printf("%3d WSASend %ld bytes\n", c->socket, n);
/* TODO handle partial sends */
c->sendbuf.len = 0;
}
return 0;
}
SOCKET make_server(int port){
int yes = 1;
struct sockaddr_in addr;
SOCKET sock;
WSADATA wsaData;
WSAStartup(MAKEWORD(2, 2), &wsaData);
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_ANY);
addr.sin_port = htons(port);
sock = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)yes, sizeof(yes));
bind(sock, (struct sockaddr*)&addr, sizeof(addr));
listen(sock, SOMAXCONN);
return sock;
}
int main(){
const char *text =
"HTTP/1.0 200 OK\r\n"
"Content-Length: 13\r\n"
"Content-Type: text/html\r\n"
"Connection: Close\r\n"
"\r\n"
"Hello, World!";
SOCKET server_socket = make_server(8080);
HANDLE iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
CreateThread(NULL, 0, worker_thread, iocp, 0, NULL);
while (1){
DWORD flags = 0, n = 0;
client *c;
struct sockaddr_in addr;
int addrlen = sizeof(addr);
SOCKET client_socket = WSAAccept(server_socket, (struct sockaddr*)&addr, &addrlen, NULL, 0);
printf("%3d connected\n", client_socket);
CreateIoCompletionPort((HANDLE)client_socket, iocp, 0, 0);
c = (client*)calloc(1, sizeof(*c));
c->socket = client_socket;
c->sendbuf.len = strlen(text);
c->recvbuf.len = BUFFER_SIZE;
c->sendbuf.buf = c->sendbuffer;
c->recvbuf.buf = c->recvbuffer;
strcpy(c->sendbuf.buf, text);
/* for some reason I have to receive 0 bytes once */
WSARecv(c->socket, &(c->recvbuf), 1, &n, &flags, &(c->overlapped), NULL);
}
}
Example output:
/* Browser makes two tcp connections on socket 124 and 128. */
124 connected
128 connected
/* GetQueuedCompletionStatus returned for socket 124. */
124 triggered
/* We received the browser's http request. */
124 WSARecv 375 bytes
/* Send http response to browser. */
124 WSASend 96 bytes
/* GetQueuedCompletionStatus returned again. */
124 triggered
/* This is wrong, we should not receive our response to the browser. */
/* Also we didn't even receive data here. */
/* recvbuffer still contains the http request. */
124 WSARecv 96 bytes
/* this is ok */
124 WSASend 0 bytes
124 triggered
124 disconnected
/* Why does GetQueuedCompletionStatus still return? the socket is closed! */
/* Also how can I tell when I can safely free the buffers */
/* if GetQueuedCompletionStatus keeps returning? */
-1 triggered
-1 GetQueuedCompletionStatus error 1236
-1 triggered
-1 GetQueuedCompletionStatus error 1236
/* same again for second http request */
128 triggered
128 WSARecv 375 bytes
128 WSASend 96 bytes
128 triggered
128 WSARecv 96 bytes
128 WSASend 0 bytes
128 triggered
128 disconnected
-1 triggered
-1 GetQueuedCompletionStatus error 1236
-1 triggered
-1 GetQueuedCompletionStatus error 1236
128 connected
128 triggered
128 WSARecv 375 bytes
128 WSASend 96 bytes
128 triggered
128 WSARecv 96 bytes
128 WSASend 0 bytes
128 triggered
128 disconnected
-1 triggered
-1 GetQueuedCompletionStatus error 1236
-1 triggered
-1 GetQueuedCompletionStatus error 1236
128 connected
128 triggered
128 WSARecv 289 bytes
128 WSASend 96 bytes
128 triggered
128 WSARecv 96 bytes
128 WSASend 0 bytes
128 triggered
128 disconnected
-1 triggered
-1 GetQueuedCompletionStatus error 1236
-1 triggered
-1 GetQueuedCompletionStatus error 1236
Your pseudo code workflow should look more like this instead:
main {
create server socket
create io completion port
create worker thread
while not done {
accept client socket
associate client socket with completion port
create recv, send, and work buffers for client
call WSARecv with >0 bytes to start filling recv buffer
if failed {
close client socket and free associated buffers
}
}
terminate worker thread
close client sockets
close server socket
}
worker thread {
while not terminated {
call GetQueuedCompletionStatus
if failed {
if failed because of IO error {
close socket and free associated buffers
}
else if not timeout {
handle error as needed
}
}
else if no bytes were transferred {
close socket and free associated buffers
}
else if IO was WSARecv {
move data from recv buffer to end of work buffer
while work buffer has a complete message {
remove message from front of work buffer, process as needed
if output to send {
if send buffer not empty {
append output to end of send buffer, will send later
}
else {
move output to send buffer
call WSASend
if failed {
close socket and free associated buffers
}
}
}
}
call WSARecv with >0 bytes to start filling recv buffer
if failed {
close socket and free associated buffers
}
}
else if IO was WSASend {
remove reported number of bytes from front of send buffer
if send buffer not empty {
call WSASend
if failed {
close socket and free associated buffers
}
}
}
}
}
I will leave it as an exercise for you to translate that into your code.