一 基本示例

#include <stdio.h>
#include <sys/socket.h>  // socket()
#include <arpa/inet.h>   // inet_addr()
#include <netinet/in.h>  // sockaddr_in{} INADDR_ANY
#include <unistd.h>      // close()
#include <errno.h>       // errno
#include <string.h>      // strerror()
#include <stdbool.h>     // true

int main(){
  
  int bYes=1;
  struct sockaddr_in inAddr;
  
  printf("\n *** cpl time [%s]  *** \n",__TIME__);
  
  inAddr.sin_family = AF_INET;
  inAddr.sin_port = htons(10240);
  //inAddr.sin_addr.s_addr = inet_addr("0.0.0.0");
  inAddr.sin_addr.s_addr = inet_addr("127.0.0.1");
  //inAddr.sin_addr.s_addr = inet_addr("192.168.28.56");
  
  int listenfd = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP);
  if (listenfd < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEPORT,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  

  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  
  if(0 > listen(listenfd,1)) {
    printf("listen [%d][%s]",errno, strerror(errno));
    return -1;
  }

  while(1) {
  
      struct sockaddr_in recvAddr;
      socklen_t stulen = sizeof(struct sockaddr);
      
      int fd = accept(listenfd,(struct sockaddr*)&recvAddr,&stulen);
      
      struct linger so_linger = {
          .l_onoff = true,    // 开启 linger 控制
          .l_linger = 0       // close_wait 时间 0s  
      };
      
      setsockopt(fd,SOL_SOCKET,SO_LINGER,&so_linger,sizeof(so_linger));
      
      if( fd > 0 ) {
      
          char buf[1024] = {0};
          
          int bt = recv(fd,buf,sizeof(buf),0);
          
          printf("recv [%d] from  [%s] :\n%s",bt,inet_ntoa(recvAddr.sin_addr),buf);
          
          send(fd,buf,sizeof(buf),0);
          
          close(fd);
          
      } else {
          printf("err [%d][%s]", errno, strerror(errno));
      }
      
  }  
  
  close(listenfd);
  return 0;
}

二 参数解析

2.1 结构体

struct sockaddr
内核 tcp / ip 协议栈实现：

// include\uapi\linux\socket.h
#define _K_SS_MAXSIZE	128	/* Implementation specific max size (see RFC2553) */
typedef unsigned short __kernel_sa_family_t;
// 这里本来 使用 匿名联合体 保证 内存对齐，这里简化了
struct __kernel_sockaddr_storage {
		struct {
			__kernel_sa_family_t	ss_family; /* address family */
			char __data[_K_SS_MAXSIZE - sizeof(unsigned short)];
		};
};

POSIX API 实现：

//include\linux\socket.h
typedef __kernel_sa_family_t	sa_family_t;
//1003.1g requires sa_family_t and that sa_data is char. 
struct sockaddr {
	sa_family_t	sa_family;	/* address family, AF_xxx	*/
	char sa_data_min[14];		/* Minimum 14 bytes of protocol address	*/
};

struct sockaddr_in

//include\uapi\linux\in.h
struct in_addr { __be32	s_addr; };

#define __SOCK_SIZE__	16		/* sizeof(struct sockaddr)	*/
struct sockaddr_in {
  __kernel_sa_family_t	sin_family;	/* Address family		*/
  __be16		sin_port;	/* Port number			*/
  struct in_addr	sin_addr;	/* Internet address		*/
  /* Pad to size of `struct sockaddr'. */
  unsigned char		__pad[__SOCK_SIZE__ - sizeof(short int) -
			sizeof(unsigned short int) - sizeof(struct in_addr)];
};

socket 不仅可以绑定 AF_INET 协议簇，还有 AF_UNIX 等各种通信域 ( communication domain )；sockaddr.sa_data 含义会随 通信域 变化；sockaddr_in 就是 sockaddr 在 AF_INET 域内特化的形式；
使用 sockaddr_in 时要注意 保持网络字节序，big-endian，或者 bigger end，低位地址更高；

//include\linux\byteorder\generic.h
#define ___htonl(x) __cpu_to_be32(x)
#define ___htons(x) __cpu_to_be16(x)
#define ___ntohl(x) __be32_to_cpu(x)
#define ___ntohs(x) __be16_to_cpu(x)

字符串 到 整型 的转换可以使用 glibc 库函数；

int inet_aton (const char *name, struct in_addr *addr)
# MT-Safe locale | AS-Safe | AC-Safe
uint32_t inet_addr (const char *name)
# MT-Safe locale | AS-Safe | AC-Safe 

listen() 将 sockfd 标记为被动套接字，仅作为 accept 的入参 监听 来访连接；

SYNOPSIS
       int listen(int sockfd, int backlog);
DESCRIPTION
       listen() marks the socket referred to by sockfd as a passive socket, that is, as a socket that will be used to accept
       incoming connection requests using accept(2).
       The sockfd argument is a file descriptor that refers to a socket of type SOCK_STREAM or SOCK_SEQPACKET.
       The backlog argument defines the maximum length to which the queue of pending connections for sockfd may grow.  If  a
       connection request arrives when the queue is full, the client may receive an error with an indication of ECONNREFUSED
       or, if the underlying protocol supports retransmission, the request may be ignored so that a later reattempt at  con‐
       nection succeeds.
RETURN VALUE
       On success, zero is returned.  On error, -1 is returned, and errno is set to indicate the error.

几个常用 Inet 地址的内核实现；

//include\uapi\linux\in.h
/* Address to accept any incoming messages. */
#define	INADDR_ANY		((unsigned long int) 0x00000000)

/* Address to send to all hosts. */
#define	INADDR_BROADCAST	((unsigned long int) 0xffffffff)

/* Network number for local host loopback. */
#define	IN_LOOPBACKNET		127

/* Address to loopback in software to local host.  */
#define	INADDR_LOOPBACK		0x7f000001	/* 127.0.0.1   */
#define	IN_LOOPBACK(a)		((((long int) (a)) & 0xff000000) == 0x7f000000)

2.2 setsockopt

2.2.1 SO_REUSEADDR

一台设备，可以坐拥多个 IP（还有 127.0.0.1 环回口）；对于 AF_INET 套接字，INADDR_ANY（0.0.0.0）作为 IP 通配符，能够指代当前设备所有 SrcIP（包括 127.0.0.1 和 192.168.*.*）；

 man 7 socket
 
SO_REUSEADDR
              Indicates that the rules used in validating addresses supplied in a bind(2) call should allow reuse of local addresses.  For AF_INET
              sockets this means that a socket may bind, except when there is an active listening socket bound to the address.  When the listening
              socket  is bound to INADDR_ANY with a specific port then it is not possible to bind to this port for any local address.  Argument is
              an integer boolean flag.

SrcPort 一旦绑定到 活动的监听套接字 上，就无法再与其他套接字绑定；
SrcPort 绑定到 INADDR_ANY 上后就 无法 再绑定到 任何 本地其他 SrcIP 上；
SO_REUSEADDR 选项能够 打破这种限制 并运用在以下四种场景中；

2.2.1.1 非活动的监听套接字

• SrcPort 没有 绑定到 活动的 监听套接字上时，开启 SO_REUSEADDR后，能够被再此绑定到 新监听套接字上；
• 什么时候会出现 “ SrcPort 没有绑定到非活动的监听套接字上¹ 这种场景？：
  step 1. 启动了一个监听服务器；
  step 2. 接收 并 派生一个子进程处理请求（代理、负载均衡）；
  step 3. 监听服务器 因 崩溃 等原因 终止运行；
  step 4. 守护进程 或 脚本重启 监听服务器，此时 SrcPort 正被子进程占用，但 SrcPort 并没有绑定到监听套接字；
  如果没有开启 SO_REUSEADDR，step 4 重启的监听服务器 会因为 SrcPort 被占用而导致 bind() 失败；

2.2.1.2 同一个端口多个服务器实例

• 理论上只要每个服务器的 SrcIP 不同，如 INADDR_ANY 和 loopback（127.0.0.1），那么就应该允许重复 bind；
• 然而实验结果打脸了… 树莓 pi 4 + Ubuntu，即便 设置了 SO_REUSEADDR 同样 bind 失败；

// server1 先启动
  int bNo = 0;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = INADDR_ANY; // = inet_addr("0.0.0.0");
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...
// server2 后启动
  int bYes = 1;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = inet_addr("127.0.0.1"); // 或者 192.168.28.56
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...

> ./server1 &
> ./server2 &
bind [98][Address already in use] # bind 失败

目前 《UNIX网络编程 卷一》给出的解释是² “很多操作系统已经不允许 对已经绑定了通配地址的端口 再绑定更为明确的地址”，为了防止某些 恶意服务 强行 “劫持” 正在提供服务的端口；所以即便设置了 SO_REUSEADDR，上述 server2 中的 127.0.0.1 依旧会绑定失败；
但问题是，我先启动的 server1 绑定到 127.0.0.1，后启动 server2 绑定到 INADDR_ANY，应该没有问题才对呀？
原因是啥？求求大佬们指条明路吧，这个问题让我备受煎熬 …

2.2.1.3 同一个端口多个IP

• 实验的结果是 即便没有设置 SO_REUSEADDR，效果相同；

// server1 先启动
  int bNo = 0;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = inet_addr("192.168.28.56"); // 注意 IP
#if 0
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
#endif
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...
// server2 后启动
  int bYes = 1;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = inet_addr("127.0.0.1"); // 注意 IP
#if 0
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
#endif
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...

> ./server1 &
> ./server2 &# bind 成功

2.2.1.4 完全重复的捆绑（completely duplicate binding）

• 目前常见协议中，仅 UDP 协议支持；
• 这个特性主要用于多播 （multicast）时，同一个主机上同时运行一个应用程序的多个副本；UDP 数据包是 多播（或者是 广播 broadcast）时，给每个套接字送一个副本；如果是单播，就只给单个套接字；详见 《UNIX网络编程 卷一》，这里不做讨论；

2.3 SO_REUSEPORT

	// ...
	if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEPORT,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  	}
  	// ...

 man 7 socket
  
SO_REUSEPORT (since Linux 3.9)
              Permits multiple AF_INET or AF_INET6 sockets to be bound to an identical socket address.  This option must be  set  on  each  socket
              (including the first socket) prior to calling bind(2) on the socket.  To prevent port hijacking, all of the processes binding to the
              same address must have the same effective UID.  This option can be employed with both TCP and UDP sockets.

              For TCP sockets, this option allows accept(2) load distribution in a multi-threaded server to be improved by using a  distinct  lis‐
              tener  socket  for  each thread.  This provides improved load distribution as compared to traditional techniques such using a single
              accept(2)ing thread that distributes connections, or having multiple threads that compete to accept(2) from the same socket.

              For UDP sockets, the use of this option can provide better distribution of incoming datagrams to multiple processes (or threads)  as
              compared to the traditional technique of having multiple processes compete to receive datagrams on the same socket.

这个是重量级！允许 真·完全重复的捆绑；
只有一个要求，每个 套接字，都在 bind 前设置过 SO_REUSEPORT；

// server1 
  int bNo = 0;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = INADDR_ANY;
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEPORT,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...
// server2 
  int bYes = 1;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = inet_addr("192.168.28.56");
  if (setsockopt(listenfd,SOL_SOCKET,SO_REUSEPORT,&bYes,sizeof(bYes)) < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  ...

> ./server1 &
> ./server2 &
> ./server2 &
> netstat -lp | grep 10240
tcp        0      0 0.0.0.0:10240           0.0.0.0:*               LISTEN      3722/./server1
tcp        0      0 hk-desktop:10240        0.0.0.0:*               LISTEN      3670/./server2
tcp        0      0 hk-desktop:10240        0.0.0.0:*               LISTEN      3669/./server2

2.4 struct linger

struct linger so_linger = {
    .l_onoff = true,    // 开启 linger 控制
    .l_linger = 0       // close_wait 时间 0s  
};
setsockopt(fd,SOL_SOCKET,SO_LINGER,&so_linger,sizeof(so_linger));

四次挥手的最后，套接字会持续 TIME_WAIT 等待 2 个 MSL 时间后再 close 套接字；
设置 l_onoff = true 后，关闭套接字（或者进程崩溃），内核 只会等待 指定的 l_linger 时间，便抛弃 套接字 内核缓冲区中 残留的数据³，不等 2 个 MSL 时间， 也不重传；

三 过程解析

一个简单的 C-S 回显实验：
server.c：

#include <stdio.h>
#include <sys/socket.h>  // socket()
#include <arpa/inet.h>   // inet_addr()
#include <netinet/in.h>  // sockaddr_in{} INADDR_ANY
#include <unistd.h>      // close()
#include <errno.h>       // errno
#include <string.h>      // strerror()
#include <stdbool.h>     // true
#include <stdlib.h>	 // exit()

int main(){
  
  int bYes=1;
  int cnt = 4;
  struct sockaddr_in inAddr;
  
  printf("\n *** cpl time [%s]  *** \n",__TIME__);
  
  inAddr.sin_family = AF_INET;
  inAddr.sin_port = htons(10240);
  inAddr.sin_addr.s_addr = INADDR_ANY;
  
  int listenfd = socket(AF_INET,SOCK_STREAM|SOCK_CLOEXEC,IPPROTO_TCP); // fork 子进程时自动关闭 listenfd
  if (listenfd < 0) {
    printf("[%d][%s]",errno, strerror(errno));
    return -1;
  }
  
  setsockopt(listenfd,SOL_SOCKET,SO_REUSEPORT,&bYes,sizeof(bYes));

  if(0 > bind(listenfd,(struct sockaddr*)&inAddr,sizeof(struct sockaddr))) {
    printf("bind [%d][%s]",errno, strerror(errno));
    return -1;
  }
  
  if(0 > listen(listenfd,5)) {
    printf("listen [%d][%s]",errno, strerror(errno));
    return -1;
  }

  while(1) {
  
      struct sockaddr_in recvAddr;
      socklen_t stulen = sizeof(struct sockaddr);
      int fd = accept(listenfd,(struct sockaddr*)&recvAddr,&stulen);
      if( fd <= 0 ) continue;
      
      // child
      if(fork()==0) {
          	printf("sub server [%d] connect from [%s]\n",getpid(),inet_ntoa(recvAddr.sin_addr));
      	 	char szbuf[1024] = {0};
			int n = 0;
        	while( (n = recv(fd,szbuf,sizeof(szbuf),0)) > 0) {
        		printf("sub server [%d], recv [%d] from [%s]:%s\n",
        		        	getpid(),n,inet_ntoa(recvAddr.sin_addr),szbuf);
        		send(fd,szbuf,n); // 回显
        	}
        	
        	while(cnt--){
        		printf("sub server [%d] [%d]s before close fd [%d]\n", getpid(),cnt,fd);
        		sleep(1);
        	}
        	close(fd);
        	exit(0);
      }

	  // parent
	  printf("\nthis is listener, continue ...\n");
  }  
  
  close(listenfd);
  return 0;
}

client.c：

#include <stdio.h>
#include <sys/socket.h>  // socket()
#include <arpa/inet.h>   // inet_addr()
#include <netinet/in.h>  // sockaddr_in{} INADDR_ANY
#include <unistd.h>      // close()
#include <string.h>

int main() {
	char szBuf[1024]={0};
	struct sockaddr_in toAddr;
	printf("\n *** cpl time [%s]  *** \n",__TIME__);
	int sockfd = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP);
	
	toAddr.sin_family = AF_INET;
	toAddr.sin_port = htons(10240);
	toAddr.sin_addr.s_addr = INADDR_ANY;
	
	connect(sockfd ,(struct sockaddr *)&toAddr,sizeof(toAddr));
	
	while(fgets(szBuf,sizeof(szBuf),stdin) != NULL) { //
		
		write(sockfd,szBuf,strlen(szBuf));
		printf("client send to [0.0.0.0]: %s",szBuf);
		memset(szBuf,0,sizeof(szBuf));
	
		memset(szBuf,0,sizeof(szBuf));
		read(sockfd,szBuf,sizeof(szBuf));
		printf("client recv from [0.0.0.0]: %s",szBuf);
	}
	
 	return 0;
}

运行 server 和 client，并在过程中使用 netstat 和 ps 命令，查看 进程 和 套接字 的状态：

$$$$$$ gcc server.c -o server
$$$$$$ ./server &
[3] 6287
 *** cpl time [14:08:08]  ***
$$$$$$ netstat pl |grep 10240
tcp        1      0 localhost:42602         localhost:10240         CLOSE_WAIT
$$$$$$ ps -o pid,args,stat,wchan
   PID COMMAND                  STAT    WCHAN # wchan 参数表明当前进程 sleep 的内核具体函数命
   5564 -bash                     Ss  	do_wait
   6552 ./server                  S 	inet_csk_accept # 服务端正阻塞在 accpet
   6626 ps -o pid,args,stat,wchan       -	  # '-' 表示 当前进程正在 running 
$$$$$$ gcc client.c -o client
$$$$$$ ./client.out
 *** cpl time [14:08:31]  ***				# client 完三次握手第二步，收到 SCK 包后 阻塞在 fgets
this is listener, continue ... 				# server 完成三次握手，accept 返回 调用 fork
sub server [6334] connect from [127.0.0.1]  # fork() 得到 sub server 子进程
123456
client send to [0.0.0.0]: 123456 # 客户端发送 
sub server [6334], recv [7] from [127.0.0.1]:123456 # sub server 发送回显
client recv from [0.0.0.0]: 123456 # 客户端收到回显
^D 	# client 收到 EOF, client 退出前关闭套接字 向 subserver 发送四次挥手的第一个 FIN 包
$$$$$$ sub server [6334] [4]s before close fd [4] 
$$$$$$ sub server [6334] [3]s before close fd [4]
$$$$$$ sub server [6334] [2]s before close fd [4]
$$$$$$ netstat -ap |grep 10240 # 此时查看 套接字 状态
tcp        1      0 localhost:42602         localhost:10240         CLOSE_WAIT
tcp        0      0 localhost:39208         localhost:10240         FIN_WAIT2   -
# client 进程已退出，套接字(port=39208) 进入 FIN_WAIT2
tcp        0      0 localhost:10240         localhost:39208         CLOSE_WAIT  7364/./server
# subserver 收到 FIN 包, recv 返回 0, 向 client 回 ACK, 进入 CLOSE_WAIT
$$$$$$ sub server [6334] [1]s before close fd [4]
$$$$$$ sub server [6334] [0]s before close fd [4]
# subserver close(fd), 向 client 回 FIN, 进入 LAST_ACK, 子进程回 ACK 后应当进入 TIME_WAIT 状态
$$$$$$ ps -o pid,args,stat,wchan |grep server
   PID COMMAND                  STAT    WCHAN # wchan 参数表明当前进程 sleep 的内核具体函数命
   5564 -bash                     Ss  	do_wait
   6552 ./server                  S 	inet_csk_accept
   6334 [server] <defunct>        Z     -	  # server 没有调用 wait() sub server 成了“僵死”进程
   6626 ps -o pid,args,stat,wchan       -	  # '-' 表示 当前进程正在 running  

---

1. UNIX 网络编程 卷一：套接字联网 API 第三版 P165 ↩︎

2. UNIX 网络编程 卷一：套接字联网 API 第三版 P166 ↩︎

3. https://www.cnblogs.com/schips/p/12553321.html ↩︎