Modify ocserv-sm to permit it to scale up to the number of CPUs. This permits a higher rate of client connections and prevents TLS signing from becoming a bottleneck for clients connecting.

Resolves: #341

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

NEWS

@@ -10,6 +10,10 @@
   a client with an RSA key (#318)
 - Enable a race free user disconnection via occtl (#59)
 - Added the config option of a pre-login-banner (#313)
+- Ocserv siwtched to using multiple ocserv-sm processes to improve scale,
+  with the number of ocserv-sm process dependent on maximum clients and
+  number of CPUs. Configuration option sec-mod-scale can be used to override
+  the heuristics.
 
 
 * Version 1.1.0 (released 2020-06-16)

src/config.c

@@ -827,6 +827,9 @@ static int cfg_ini_handler(void *_ctx, const char *section, const char *name, co
 				READ_STATIC_STRING(pid_file);
 			} else if (reload == 0)
 				fprintf(stderr, NOTESTR"skipping 'pid-file' config option\n");
+		} else if (strcmp(name, "sec-mod-scale") == 0) {
+			if (!PWARN_ON_VHOST(vhost->name, "sec-mod-scale", sec_mod_scale))
+				READ_NUMERIC(vhost->perm_config.sec_mod_scale);
 		} else {
 			stage1_found = 0;
 		}

src/ctl.proto

@@ -7,7 +7,7 @@ message status_rep
 {
 	required bool status = 1;
 	required uint32 pid = 2;
-	required uint32 sec_mod_pid = 3;
+	repeated uint32 sec_mod_pids = 3;
 	required uint32 active_clients = 4;
 	required uint32 start_time = 5;
 	required uint32 stored_tls_sessions = 7;

src/isolate.c

@@ -47,12 +47,13 @@ void update_fd_limits(main_server_st * s, unsigned main)
 	int ret;
 
 	if (main) {
-		if (GETCONFIG(s)->max_clients > 0
+		if (GETCONFIG(s)->max_clients > 0) 
-		    && GETCONFIG(s)->max_clients >
+			// FUTURE: Should this be raises to account for scripts?
-		    s->fd_limits_default_set.rlim_cur)
+			max = GETCONFIG(s)->max_clients + 32 + s->sec_mod_instance_count * 2;
-			max = GETCONFIG(s)->max_clients + 32;
 		else
-			max = MAX(4 * 1024, s->fd_limits_default_set.rlim_cur);
+			// If the admin doesn't specify max_clients,
+			// then we are limiting it to around 4K.
+			max = 4 * 1024;
 
 		if (max > s->fd_limits_default_set.rlim_cur) {
 			new_set.rlim_cur = max;

src/main-auth.c

@@ -163,9 +163,10 @@ int send_cookie_auth_reply(main_server_st* s, struct proc_st* proc,
 	return 0;
 }
 
-int handle_auth_cookie_req(main_server_st* s, struct proc_st* proc,
+int handle_auth_cookie_req(sec_mod_instance_st * sec_mod_instance, struct proc_st* proc,
  			   const AuthCookieRequestMsg * req)
 {
+	main_server_st * s = sec_mod_instance->server;
 	int ret;
 	struct proc_st *old_proc;
 
@@ -189,7 +190,7 @@ int handle_auth_cookie_req(main_server_st* s, struct proc_st* proc,
 	}
 
 	/* loads sup config and basic proc info (e.g., username) */
-	ret = session_open(s, proc, req->cookie.data, req->cookie.len);
+	ret = session_open(sec_mod_instance, proc, req->cookie.data, req->cookie.len);
 	if (ret < 0) {
 		mslog(s, proc, LOG_INFO, "could not open session");
 		return -1;

src/main-ctl-unix.c

@@ -174,16 +174,39 @@ static void method_status(method_ctx *ctx, int cfd, uint8_t * msg,
 {
 	StatusRep rep = STATUS_REP__INIT;
 	int ret;
+	unsigned int i;
+	uint32_t * sec_mod_pids;
+
+	sec_mod_pids = talloc_array(ctx->pool, uint32_t, ctx->s->sec_mod_instance_count);
+	if (sec_mod_pids) {
+		for (i = 0; i < ctx->s->sec_mod_instance_count; i ++) {
+			sec_mod_pids[i] = ctx->s->sec_mod_instances[i].sec_mod_pid;
+		}
+	}
 
 	mslog(ctx->s, NULL, LOG_DEBUG, "ctl: status");
 
 	rep.status = 1;
 	rep.pid = getpid();
 	rep.start_time = ctx->s->stats.start_time;
-	rep.sec_mod_pid = ctx->s->sec_mod_pid;
+	if (sec_mod_pids) {
+		rep.sec_mod_pids = sec_mod_pids;
+		rep.n_sec_mod_pids = ctx->s->sec_mod_instance_count;
+	}
 	rep.active_clients = ctx->s->stats.active_clients;
-	rep.secmod_client_entries = ctx->s->stats.secmod_client_entries;
+	rep.secmod_client_entries = 0;
-	rep.stored_tls_sessions = ctx->s->stats.tlsdb_entries;
+	rep.stored_tls_sessions = 0;
+	rep.max_auth_time = 0;
+	rep.avg_auth_time = 0;
+	for (i = 0; i < ctx->s->sec_mod_instance_count; i ++) {
+		rep.secmod_client_entries += ctx->s->sec_mod_instances[i].secmod_client_entries;
+		rep.stored_tls_sessions += ctx->s->sec_mod_instances[i].tlsdb_entries;
+		rep.max_auth_time = MAX(rep.max_auth_time, ctx->s->sec_mod_instances[i].max_auth_time);
+		rep.avg_auth_time = ctx->s->sec_mod_instances[i].avg_auth_time;
+	}
+	if (ctx->s->sec_mod_instance_count != 0) {
+		rep.avg_auth_time /= ctx->s->sec_mod_instance_count;
+	}
 	rep.banned_ips = main_ban_db_elems(ctx->s);
 
 	rep.session_timeouts = ctx->s->stats.session_timeouts;
@@ -195,9 +218,7 @@ static void method_status(method_ctx *ctx, int cfd, uint8_t * msg,
 	rep.min_mtu = ctx->s->stats.min_mtu;
 	rep.max_mtu = ctx->s->stats.max_mtu;
 	rep.last_reset = ctx->s->stats.last_reset;
-	rep.avg_auth_time = ctx->s->stats.avg_auth_time;
 	rep.avg_session_mins = ctx->s->stats.avg_session_mins;
-	rep.max_auth_time = ctx->s->stats.max_auth_time;
 	rep.max_session_mins = ctx->s->stats.max_session_mins;
 
 	rep.auth_failures = ctx->s->stats.auth_failures;
@@ -551,22 +572,86 @@ static void method_list_banned(method_ctx *ctx, int cfd, uint8_t * msg,
 static void method_list_cookies(method_ctx *ctx, int cfd, uint8_t * msg,
 			      unsigned msg_size)
 {
+	SecmListCookiesReplyMsg reply = SECM_LIST_COOKIES_REPLY_MSG__INIT;
+	SecmListCookiesReplyMsg ** sub_replies = NULL;
+	CookieIntMsg ** cookies = NULL;
+	PROTOBUF_ALLOCATOR(pa, ctx->pool);
+
+	size_t total_cookies = 0;
+	unsigned int i;
+	unsigned int j;
+	unsigned int k;
 	int ret;
 
 	mslog(ctx->s, NULL, LOG_DEBUG, "ctl: list-cookies");
 
-	ret = send_msg(ctx->pool, ctx->s->sec_mod_fd_sync, CMD_SECM_LIST_COOKIES,
+	sub_replies = talloc_zero_array(ctx->pool, SecmListCookiesReplyMsg*, ctx->s->sec_mod_instance_count);
-			 NULL, NULL, NULL);
+	if (!sub_replies) {
-	if (ret < 0) {
+		goto reply_and_exit;
-		mslog(ctx->s, NULL, LOG_ERR, "error sending list cookies to sec-mod!");
 	}
 
-	ret = forward_msg(ctx->pool, ctx->s->sec_mod_fd_sync, CMD_SECM_LIST_COOKIES_REPLY,
+	for (i = 0; i < ctx->s->sec_mod_instance_count; i++) {
-			  cfd, CTL_CMD_LIST_COOKIES_REP, MAIN_SEC_MOD_TIMEOUT);
+		SecmListCookiesReplyMsg * sub_reply = NULL;
+		ret = send_msg(ctx->pool, ctx->s->sec_mod_instances[i].sec_mod_fd_sync, CMD_SECM_LIST_COOKIES,
+				NULL, NULL, NULL);
+		if (ret < 0) {
+			mslog(ctx->s, NULL, LOG_ERR, "error sending list cookies to sec-mod!");
+			continue;
+		}
+		ret = recv_msg(ctx->pool, ctx->s->sec_mod_instances[i].sec_mod_fd_sync, CMD_SECM_LIST_COOKIES_REPLY, 
+				(void*)&sub_reply, (unpack_func)secm_list_cookies_reply_msg__unpack, MAIN_SEC_MOD_TIMEOUT);
+		if (ret < 0) {
+			mslog(ctx->s, NULL, LOG_ERR, "error receiving list cookies reply");
+			continue;
+		}
+		
+		if (sub_reply) {
+			sub_replies[i] = sub_reply;
+			total_cookies += sub_reply->n_cookies;
+		}
+	}
+
+	cookies = talloc_zero_array(ctx->pool, CookieIntMsg*, total_cookies);
+	if (!cookies) {
+		goto reply_and_exit;
+	}
+
+	k = 0;
+	for (i = 0; i < ctx->s->sec_mod_instance_count; i++) {
+		if (sub_replies[i] == NULL) {
+			continue;
+		}
+
+		for (j = 0; j < sub_replies[i]->n_cookies; j++) {
+			cookies[k++] = sub_replies[i]->cookies[j];
+		}
+	}
+
+reply_and_exit:
+	reply.cookies = cookies;
+	reply.n_cookies = total_cookies;
+
+	ret = send_msg(ctx->pool, cfd, CTL_CMD_LIST_COOKIES_REP, &reply,
+		       (pack_size_func) secm_list_cookies_reply_msg__get_packed_size,
+		       (pack_func) secm_list_cookies_reply_msg__pack);
 	if (ret < 0) {
 		mslog(ctx->s, NULL, LOG_ERR, "error sending list cookies reply");
 	}
 
+	if (sub_replies) {
+		for (i = 0; i < ctx->s->sec_mod_instance_count; i++) {
+			if (sub_replies[i] == NULL) {
+				continue;
+			}
+			secm_list_cookies_reply_msg__free_unpacked(sub_replies[i], &pa);
+		}
+		talloc_free(sub_replies);
+	}
+
+	if (cookies) {
+		talloc_free(cookies);
+	}
+	return;
 }
 
 static void single_info_common(method_ctx *ctx, int cfd, uint8_t * msg,

src/main-proc.c

@@ -111,7 +111,7 @@ void remove_proc(main_server_st * s, struct proc_st *proc, unsigned flags)
 
 	/* close any pending sessions */
 	if (proc->active_sid && !(flags & RPROC_QUIT)) {
-		if (session_close(s, proc) < 0) {
+		if (session_close(&(s->sec_mod_instances[proc->sec_mod_instance_index]), proc) < 0) {
 			mslog(s, proc, LOG_ERR, "error closing session (communication with sec-mod issue)");
 			exit(1);
 		}

src/main-sec-mod-cmd.c

@@ -66,8 +66,9 @@ static void update_auth_failures(main_server_st * s, uint64_t auth_failures)
 	s->stats.total_auth_failures += auth_failures;
 }
 
-int handle_sec_mod_commands(main_server_st * s)
+int handle_sec_mod_commands(sec_mod_instance_st * sec_mod_instance)
 {
+	struct main_server_st * s = sec_mod_instance->server;
 	struct iovec iov[3];
 	uint8_t cmd;
 	struct msghdr hdr;
@@ -92,7 +93,7 @@ int handle_sec_mod_commands(main_server_st * s)
 	hdr.msg_iovlen = 2;
 
 	do {
-		ret = recvmsg(s->sec_mod_fd, &hdr, 0);
+		ret = recvmsg(sec_mod_instance->sec_mod_fd, &hdr, 0);
 	} while(ret == -1 && errno == EINTR);
 	if (ret == -1) {
 		e = errno;
@@ -122,7 +123,7 @@ int handle_sec_mod_commands(main_server_st * s)
 		return ERR_MEM;
 	}
 
-	raw_len = force_read_timeout(s->sec_mod_fd, raw, length, MAIN_SEC_MOD_TIMEOUT);
+	raw_len = force_read_timeout(sec_mod_instance->sec_mod_fd, raw, length, MAIN_SEC_MOD_TIMEOUT);
 	if (raw_len != length) {
 		e = errno;
 		mslog(s, NULL, LOG_ERR,
@@ -159,7 +160,7 @@ int handle_sec_mod_commands(main_server_st * s)
 
 			mslog(s, NULL, LOG_DEBUG, "sending msg %s to sec-mod", cmd_request_to_str(CMD_SECM_BAN_IP_REPLY));
 
-			ret = send_msg(NULL, s->sec_mod_fd, CMD_SECM_BAN_IP_REPLY,
+			ret = send_msg(NULL, sec_mod_instance->sec_mod_fd, CMD_SECM_BAN_IP_REPLY,
 				&reply, (pack_size_func)ban_ip_reply_msg__get_packed_size,
 				(pack_func)ban_ip_reply_msg__pack);
 			if (ret < 0) {
@@ -185,10 +186,10 @@ int handle_sec_mod_commands(main_server_st * s)
 				goto cleanup;
 			}
 
-			s->stats.secmod_client_entries = smsg->secmod_client_entries;
+			sec_mod_instance->secmod_client_entries = smsg->secmod_client_entries;
-			s->stats.tlsdb_entries = smsg->secmod_tlsdb_entries;
+			sec_mod_instance->tlsdb_entries = smsg->secmod_tlsdb_entries;
-			s->stats.max_auth_time = smsg->secmod_max_auth_time;
+			sec_mod_instance->max_auth_time = smsg->secmod_max_auth_time;
-			s->stats.avg_auth_time = smsg->secmod_avg_auth_time;
+			sec_mod_instance->avg_auth_time = smsg->secmod_avg_auth_time;
 			update_auth_failures(s, smsg->secmod_auth_failures);
 
 		}
@@ -210,7 +211,7 @@ int handle_sec_mod_commands(main_server_st * s)
 	return ret;
 }
 
-static void append_routes(main_server_st *s, proc_st *proc, GroupCfgSt *gc)
+static void append_routes(sec_mod_instance_st * sec_mod_instance, proc_st *proc, GroupCfgSt *gc)
 {
 	vhost_cfg_st *vhost = proc->vhost;
 
@@ -294,7 +295,7 @@ static void append_routes(main_server_st *s, proc_st *proc, GroupCfgSt *gc)
 }
 
 static
-void apply_default_config(main_server_st *s, proc_st *proc, GroupCfgSt *gc)
+void apply_default_config(sec_mod_instance_st * sec_mod_instance, proc_st *proc, GroupCfgSt *gc)
 {
 	vhost_cfg_st *vhost = proc->vhost;
 
@@ -308,7 +309,7 @@ void apply_default_config(main_server_st *s, proc_st *proc, GroupCfgSt *gc)
 		gc->n_routes = vhost->perm_config.config->network.routes_size;
 	}
 
-	append_routes(s, proc, gc);
+	append_routes(sec_mod_instance, proc, gc);
 
 	if (gc->no_routes == NULL) {
 		gc->no_routes = vhost->perm_config.config->network.no_routes;
@@ -447,9 +448,10 @@ void apply_default_config(main_server_st *s, proc_st *proc, GroupCfgSt *gc)
 	(*proc->config_usage_count)++;
 }
 
-int session_open(main_server_st *s, struct proc_st *proc, const uint8_t *cookie, unsigned cookie_size)
+int session_open(sec_mod_instance_st * sec_mod_instance, struct proc_st *proc, const uint8_t *cookie, unsigned cookie_size)
 {
 	int ret, e;
+	main_server_st * s = sec_mod_instance->server;
 	SecmSessionOpenMsg ireq = SECM_SESSION_OPEN_MSG__INIT;
 	SecmSessionReplyMsg *msg = NULL;
 	char str_ipv4[MAX_IP_STR];
@@ -476,7 +478,7 @@ int session_open(main_server_st *s, struct proc_st *proc, const uint8_t *cookie,
 
 	mslog(s, proc, LOG_DEBUG, "sending msg %s to sec-mod", cmd_request_to_str(CMD_SECM_SESSION_OPEN));
 
-	ret = send_msg(proc, s->sec_mod_fd_sync, CMD_SECM_SESSION_OPEN,
+	ret = send_msg(proc, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_SESSION_OPEN,
 		&ireq, (pack_size_func)secm_session_open_msg__get_packed_size,
 		(pack_func)secm_session_open_msg__pack);
 	if (ret < 0) {
@@ -485,7 +487,7 @@ int session_open(main_server_st *s, struct proc_st *proc, const uint8_t *cookie,
 		return -1;
 	}
 
-	ret = recv_msg(proc, s->sec_mod_fd_sync, CMD_SECM_SESSION_REPLY,
+	ret = recv_msg(proc, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_SESSION_REPLY,
 	       (void *)&msg, (unpack_func) secm_session_reply_msg__unpack, MAIN_SEC_MOD_TIMEOUT);
 	if (ret < 0) {
 		e = errno;
@@ -533,7 +535,7 @@ int session_open(main_server_st *s, struct proc_st *proc, const uint8_t *cookie,
 	proc->vhost = find_vhost(s->vconfig, msg->vhost);
 
 	if (proc->config) {
-		apply_default_config(s, proc, proc->config);
+		apply_default_config(sec_mod_instance, proc, proc->config);
 
 		/* check whether the cookie IP matches */
 		if (proc->config->deny_roaming != 0) {
@@ -558,6 +560,17 @@ int session_open(main_server_st *s, struct proc_st *proc, const uint8_t *cookie,
 
 static void reset_stats(main_server_st *s, time_t now)
 {
+	unsigned int i;	
+	unsigned long max_auth_time = 0;
+	unsigned long avg_auth_time = 0;
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		max_auth_time = MAX(max_auth_time, s->sec_mod_instances[i].max_auth_time);
+		s->sec_mod_instances[i].max_auth_time = 0;
+		avg_auth_time += s->sec_mod_instances[i].avg_auth_time;
+		s->sec_mod_instances[i].avg_auth_time = 0;
+	}
+	if (s->sec_mod_instance_count != 0)
+		avg_auth_time /= s->sec_mod_instance_count;
 	mslog(s, NULL, LOG_INFO, "Start statistics block");
 	mslog(s, NULL, LOG_INFO, "Total sessions handled: %lu", (unsigned long)s->stats.total_sessions_closed);
 	mslog(s, NULL, LOG_INFO, "Sessions handled: %lu", (unsigned long)s->stats.sessions_closed);
@@ -569,8 +582,8 @@ static void reset_stats(main_server_st *s, time_t now)
 
 	mslog(s, NULL, LOG_INFO, "Total authentication failures: %lu", (unsigned long)s->stats.total_auth_failures);
 	mslog(s, NULL, LOG_INFO, "Authentication failures: %lu", (unsigned long)s->stats.auth_failures);
-	mslog(s, NULL, LOG_INFO, "Maximum authentication time: %lu sec", (unsigned long)s->stats.max_auth_time);
+	mslog(s, NULL, LOG_INFO, "Maximum authentication time: %lu sec", max_auth_time);
-	mslog(s, NULL, LOG_INFO, "Average authentication time: %lu sec", (unsigned long)s->stats.avg_auth_time);
+	mslog(s, NULL, LOG_INFO, "Average authentication time: %lu sec", avg_auth_time);
 	mslog(s, NULL, LOG_INFO, "Data in: %lu, out: %lu kbytes", (unsigned long)s->stats.kbytes_in, (unsigned long)s->stats.kbytes_out);
 	mslog(s, NULL, LOG_INFO, "End of statistics block; resetting non-total stats");
 
@@ -583,7 +596,7 @@ static void reset_stats(main_server_st *s, time_t now)
 	s->stats.kbytes_in = 0;
 	s->stats.kbytes_out = 0;
 	s->stats.max_session_mins = 0;
-	s->stats.max_auth_time = 0;
+
 }
 
 static void update_main_stats(main_server_st * s, struct proc_st *proc)
@@ -642,8 +655,9 @@ static void update_main_stats(main_server_st * s, struct proc_st *proc)
 	reset_stats(s, now);
 }
 
-int session_close(main_server_st * s, struct proc_st *proc)
+int session_close(sec_mod_instance_st * sec_mod_instance, struct proc_st *proc)
 {
+	main_server_st * s = sec_mod_instance->server;
 	int ret, e;
 	SecmSessionCloseMsg ireq = SECM_SESSION_CLOSE_MSG__INIT;
 	CliStatsMsg *msg = NULL;
@@ -663,7 +677,7 @@ int session_close(main_server_st * s, struct proc_st *proc)
 
 	mslog(s, proc, LOG_DEBUG, "sending msg %s to sec-mod", cmd_request_to_str(CMD_SECM_SESSION_CLOSE));
 
-	ret = send_msg(proc, s->sec_mod_fd_sync, CMD_SECM_SESSION_CLOSE,
+	ret = send_msg(proc, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_SESSION_CLOSE,
 		&ireq, (pack_size_func)secm_session_close_msg__get_packed_size,
 		(pack_func)secm_session_close_msg__pack);
 	if (ret < 0) {
@@ -672,7 +686,7 @@ int session_close(main_server_st * s, struct proc_st *proc)
 		return -1;
 	}
 
-	ret = recv_msg(proc, s->sec_mod_fd_sync, CMD_SECM_CLI_STATS,
+	ret = recv_msg(proc, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_CLI_STATS,
 	       (void *)&msg, (unpack_func) cli_stats_msg__unpack, MAIN_SEC_MOD_TIMEOUT);
 	if (ret < 0) {
 		e = errno;
@@ -693,13 +707,14 @@ int session_close(main_server_st * s, struct proc_st *proc)
 	return 0;
 }
 
-int secmod_reload(main_server_st * s)
+int secmod_reload(sec_mod_instance_st * sec_mod_instance)
 {
+	main_server_st * s = sec_mod_instance->server;
 	int ret, e;
 
 	mslog(s, NULL, LOG_DEBUG, "sending msg %s to sec-mod", cmd_request_to_str(CMD_SECM_RELOAD));
 
-	ret = send_msg(s->main_pool, s->sec_mod_fd_sync, CMD_SECM_RELOAD,
+	ret = send_msg(s->main_pool, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_RELOAD,
 		       NULL, NULL, NULL);
 	if (ret < 0) {
 		mslog(s, NULL, LOG_ERR,
@@ -707,7 +722,7 @@ int secmod_reload(main_server_st * s)
 		return -1;
 	}
 
-	ret = recv_msg(s->main_pool, s->sec_mod_fd_sync, CMD_SECM_RELOAD_REPLY,
+	ret = recv_msg(s->main_pool, sec_mod_instance->sec_mod_fd_sync, CMD_SECM_RELOAD_REPLY,
 		       NULL, NULL, MAIN_SEC_MOD_TIMEOUT);
 	if (ret < 0) {
 		e = errno;
@@ -732,29 +747,32 @@ static void clear_unneeded_mem(struct list_head *vconfig)
  * The sync_fd is used by main to send synchronous commands- commands which
  * expect a reply immediately.
  */
-int run_sec_mod(main_server_st *s, int *sync_fd)
+void run_sec_mod(sec_mod_instance_st * sec_mod_instance, unsigned int instance_index)
 {
 	int e, fd[2], ret;
 	int sfd[2];
 	pid_t pid;
 	const char *p;
+	
+	main_server_st * s = sec_mod_instance->server; 
 
-	/* fills s->socket_file */
+	/* fills sec_mod_instance->socket_file */
-	strlcpy(s->socket_file, secmod_socket_file_name(GETPCONFIG(s)), sizeof(s->socket_file));
+	
-	mslog(s, NULL, LOG_DEBUG, "created sec-mod socket file (%s)", s->socket_file);
+	snprintf(sec_mod_instance->socket_file, sizeof(sec_mod_instance->socket_file), "%s.%d", secmod_socket_file_name(GETPCONFIG(s)), instance_index);
+	mslog(s, NULL, LOG_DEBUG, "created sec-mod socket file (%s)", sec_mod_instance->socket_file);
 
 	if (GETPCONFIG(s)->chroot_dir != NULL) {
-		ret = snprintf(s->full_socket_file, sizeof(s->full_socket_file), "%s/%s",
+		ret = snprintf(sec_mod_instance->full_socket_file, sizeof(sec_mod_instance->full_socket_file), "%s/%s",
-			       GETPCONFIG(s)->chroot_dir, s->socket_file);
+			       GETPCONFIG(s)->chroot_dir, sec_mod_instance->socket_file);
-		if (ret != strlen(s->full_socket_file)) {
+		if (ret != strlen(sec_mod_instance->full_socket_file)) {
-			mslog(s, NULL, LOG_ERR, "too long chroot path; cannot create socket: %s", s->full_socket_file);
+			mslog(s, NULL, LOG_ERR, "too long chroot path; cannot create socket: %s", sec_mod_instance->full_socket_file);
 			exit(1);
 		}
 	} else {
-		strlcpy(s->full_socket_file, s->socket_file, sizeof(s->full_socket_file));
+		strlcpy(sec_mod_instance->full_socket_file, sec_mod_instance->socket_file, sizeof(sec_mod_instance->full_socket_file));
 	}
 
-	p = s->full_socket_file;
+	p = sec_mod_instance->full_socket_file;
 
 	ret = socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
 	if (ret < 0) {
@@ -784,16 +802,17 @@ int run_sec_mod(main_server_st *s, int *sync_fd)
 		set_cloexec_flag (fd[0], 1);
 		set_cloexec_flag (sfd[0], 1);
 		clear_unneeded_mem(s->vconfig);
-		sec_mod_server(s->main_pool, s->config_pool, s->vconfig, p, fd[0], sfd[0], sizeof(s->hmac_key), s->hmac_key);
+		sec_mod_server(s->main_pool, s->config_pool, s->vconfig, p, fd[0], sfd[0], sizeof(s->hmac_key), s->hmac_key, instance_index);
 		exit(0);
 	} else if (pid > 0) {	/* parent */
 		close(fd[0]);
 		close(sfd[0]);
-		s->sec_mod_pid = pid;
+		sec_mod_instance->sec_mod_pid = pid;
 		set_cloexec_flag (fd[1], 1);
 		set_cloexec_flag (sfd[1], 1);
-		*sync_fd = sfd[1];
+		sec_mod_instance->sec_mod_fd_sync = sfd[1];
-		return fd[1];
+		sec_mod_instance->sec_mod_fd = fd[1];
+		return;
 	} else {
 		e = errno;
 		mslog(s, NULL, LOG_ERR, "error in fork(): %s", strerror(e));

src/main-worker-cmd.c

@@ -427,7 +427,9 @@ int handle_worker_commands(main_server_st * s, struct proc_st *proc)
 			goto cleanup;
 		}
 
-		ret = handle_auth_cookie_req(s, proc, auth_cookie_req);
+		proc->sec_mod_instance_index = auth_cookie_req->cookie.data[0] % s->sec_mod_instance_count;
+
+		ret = handle_auth_cookie_req(&s->sec_mod_instances[proc->sec_mod_instance_index], proc, auth_cookie_req);
 
 		safe_memset(raw, 0, raw_len);
 		safe_memset(auth_cookie_req->cookie.data, 0, auth_cookie_req->cookie.len);

src/main.c

@@ -94,15 +94,20 @@ sigset_t sig_default_set;
 struct ev_loop *loop = NULL;
 static unsigned allow_broken_clients = 0;
 
+typedef struct sec_mod_watcher_st {
+	ev_io sec_mod_watcher;
+	ev_child child_watcher;
+	unsigned int sec_mod_instance_index;
+} sec_mod_watcher_st;
+
 /* EV watchers */
 ev_io ctl_watcher;
-ev_io sec_mod_watcher;
+sec_mod_watcher_st * sec_mod_watchers = NULL;
 ev_timer maintenance_watcher;
 ev_signal maintenance_sig_watcher;
 ev_signal term_sig_watcher;
 ev_signal int_sig_watcher;
 ev_signal reload_sig_watcher;
-ev_child child_watcher;
 #if defined(CAPTURE_LATENCY_SUPPORT)
 ev_timer latency_watcher;
 #endif
@@ -511,6 +516,7 @@ int y;
  */
 void clear_lists(main_server_st *s)
 {
+	int i;
 	struct listener_st *ltmp = NULL, *lpos;
 	struct proc_st *ctmp = NULL, *cpos;
 	struct script_wait_st *script_tmp = NULL, *script_pos;
@@ -549,8 +555,10 @@ void clear_lists(main_server_st *s)
 	/* clear libev state */
 	if (loop) {
 		ev_io_stop (loop, &ctl_watcher);
-		ev_io_stop (loop, &sec_mod_watcher);
+		for (i = 0; i < s->sec_mod_instance_count; i++) {
-		ev_child_stop (loop, &child_watcher);
+			ev_io_stop (loop, &sec_mod_watchers[i].sec_mod_watcher);
+			ev_child_stop (loop, &sec_mod_watchers[i].child_watcher);
+		}
 		ev_timer_stop(loop, &maintenance_watcher);
 #if defined(CAPTURE_LATENCY_SUPPORT)
 		ev_timer_stop(loop, &latency_watcher);		
@@ -942,14 +950,17 @@ static void worker_child_watcher_cb(struct ev_loop *loop, ev_child *w, int reven
 static void kill_children(main_server_st* s)
 {
 	struct proc_st *ctmp = NULL, *cpos;
-
+	int i;
 	/* kill the security module server */
 	list_for_each_safe(&s->proc_list.head, ctmp, cpos, list) {
 		if (ctmp->pid != -1) {
 			remove_proc(s, ctmp, RPROC_KILL|RPROC_QUIT);
 		}
 	}
-	kill(s->sec_mod_pid, SIGTERM);
+
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		kill(s->sec_mod_instances[i].sec_mod_pid, SIGTERM);
+	}
 }
 
 static void kill_children_auth_timeout(main_server_st* s)
@@ -992,19 +1003,21 @@ static void reload_sig_watcher_cb(struct ev_loop *loop, ev_signal *w, int revent
 {
 	main_server_st *s = ev_userdata(loop);
 	int ret;
+	int i;
 
 	mslog(s, NULL, LOG_INFO, "reloading configuration");
-	kill(s->sec_mod_pid, SIGHUP);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		kill(s->sec_mod_instances[i].sec_mod_pid, SIGHUP);
 
-	/* Reload on main needs to happen later than sec-mod.
+		/* Reload on main needs to happen later than sec-mod.
-	 * That's because of a test that the certificate matches the
+		* That's because of a test that the certificate matches the
-	 * used key. */
+		* used key. */
-	ret = secmod_reload(s);
+		ret = secmod_reload(&s->sec_mod_instances[i]);
-	if (ret < 0) {
+		if (ret < 0) {
-		mslog(s, NULL, LOG_ERR, "could not reload sec-mod!\n");
+			mslog(s, NULL, LOG_ERR, "could not reload sec-mod!\n");
-		ev_feed_signal_event (loop, SIGTERM);
+			ev_feed_signal_event (loop, SIGTERM);
+		}
 	}
-
 	reload_cfg_file(s->config_pool, s->vconfig, 0);
 }
 
@@ -1039,6 +1052,7 @@ static void listen_watcher_cb (EV_P_ ev_io *w, int revents)
 	int fd, ret;
 	int cmd_fd[2];
 	pid_t pid;
+	int i;
 	hmac_component_st hmac_components[3];
 	char worker_path[_POSIX_PATH_MAX];
 
@@ -1093,6 +1107,7 @@ static void listen_watcher_cb (EV_P_ ev_io *w, int revents)
 
 		pid = fork();
 		if (pid == 0) {	/* child */
+			unsigned int sec_mod_instance_index;
 			/* close any open descriptors, and erase
 			 * sensitive data before running the worker
 			 */
@@ -1100,17 +1115,24 @@ static void listen_watcher_cb (EV_P_ ev_io *w, int revents)
 			close(cmd_fd[0]);
 			clear_lists(s);
 			if (s->top_fd != -1) close(s->top_fd);
-			close(s->sec_mod_fd);
+			for (i = 0; i < s->sec_mod_instance_count; i ++) {
-			close(s->sec_mod_fd_sync);
+				close(s->sec_mod_instances[i].sec_mod_fd);
+				close(s->sec_mod_instances[i].sec_mod_fd_sync);
+			}
 
 			setproctitle(PACKAGE_NAME"-worker");
 			kill_on_parent_kill(SIGTERM);
 
 			set_self_oom_score_adj(s);
 
+			sec_mod_instance_index = hash_any(
+				SA_IN_P_GENERIC(&ws->remote_addr, ws->remote_addr_len),
+				SA_IN_SIZE(ws->remote_addr_len), 0) % s->sec_mod_instance_count;
+
 			/* write sec-mod's address */
-			memcpy(&ws->secmod_addr, &s->secmod_addr, s->secmod_addr_len);
+			memcpy(&ws->secmod_addr, &s->sec_mod_instances[sec_mod_instance_index].secmod_addr, s->sec_mod_instances[sec_mod_instance_index].secmod_addr_len);
-			ws->secmod_addr_len = s->secmod_addr_len;
+			ws->secmod_addr_len = s->sec_mod_instances[sec_mod_instance_index].secmod_addr_len;
+
 
 			ws->main_pool = s->main_pool;
 
@@ -1203,7 +1225,7 @@ fork_failed:
 	if (GETCONFIG(s)->rate_limit_ms > 0) {
 		int rqueue = 0;
 		int wqueue = 0;
-		int retval = sockdiag_query_unix_domain_socket_queue_length(s->secmod_addr.sun_path, &rqueue, &wqueue);
+		int retval = sockdiag_query_unix_domain_socket_queue_length(s->sec_mod_instances[0].secmod_addr.sun_path, &rqueue, &wqueue);
 		mslog(s, NULL, LOG_DEBUG, "queue_length retval:%d rqueue:%d wqueue:%d", retval, rqueue, wqueue);
 		if (retval || rqueue > wqueue / 2) {
 			mslog(s, NULL, LOG_INFO, "delaying accepts for %d ms", GETCONFIG(s)->rate_limit_ms);
@@ -1217,10 +1239,11 @@ fork_failed:
 
 static void sec_mod_watcher_cb (EV_P_ ev_io *w, int revents)
 {
+	sec_mod_watcher_st *sec_mod = (sec_mod_watcher_st *)w;
 	main_server_st *s = ev_userdata(loop);
 	int ret;
 
-	ret = handle_sec_mod_commands(s);
+	ret = handle_sec_mod_commands(&s->sec_mod_instances[sec_mod->sec_mod_instance_index]);
 	if (ret < 0) { /* bad commands from sec-mod are unacceptable */
 		mslog(s, NULL, LOG_ERR,
 		       "error in command from sec-mod");
@@ -1307,6 +1330,7 @@ int main(int argc, char** argv)
 	main_server_st *s;
 	char *str;
 	int i;
+	int processor_count = 0;
 
 #ifdef DEBUG_LEAKS
 	talloc_enable_leak_report_full();
@@ -1315,6 +1339,8 @@ int main(int argc, char** argv)
 	saved_argc = argc;
 	saved_argv = argv;
 
+	processor_count = sysconf(_SC_NPROCESSORS_ONLN);
+
 	/* main pool */
 	main_pool = talloc_init("main");
 	if (main_pool == NULL) {
@@ -1440,7 +1466,30 @@ int main(int argc, char** argv)
 
 	write_pid_file();
 
-	s->sec_mod_fd = run_sec_mod(s, &s->sec_mod_fd_sync);
+	// Start the configured number of ocserv-sm processes
+	s->sec_mod_instance_count = GETPCONFIG(s)->sec_mod_scale;
+	
+	if (s->sec_mod_instance_count == 0)	{
+		if (GETCONFIG(s)->max_clients != 0) {
+			// Compute ideal number of clients per sec-mod
+			unsigned int sec_mod_count_for_users = GETCONFIG(s)->max_clients / MINIMUM_USERS_PER_SEC_MOD + 1;
+			// Limit it to number of processors. 
+			s->sec_mod_instance_count = MIN(processor_count,sec_mod_count_for_users);
+		} else {
+			// If it's unlimited, the use processor count.
+			s->sec_mod_instance_count = processor_count;
+		}
+	}
+
+	s->sec_mod_instances = talloc_zero_array(s, sec_mod_instance_st, s->sec_mod_instance_count);
+	sec_mod_watchers = talloc_zero_array(s, sec_mod_watcher_st, s->sec_mod_instance_count);
+
+	mslog(s, NULL, LOG_INFO, "Starting %d instances of ocserv-sm", s->sec_mod_instance_count);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		s->sec_mod_instances[i].server = s;
+		run_sec_mod(&s->sec_mod_instances[i], i);
+	}
+
 	ret = ctl_handler_init(s);
 	if (ret < 0) {
 		mslog(s, NULL, LOG_ERR, "Cannot create command handler");
@@ -1466,12 +1515,14 @@ int main(int argc, char** argv)
 	}
 	ms_sleep(100); /* give some time for sec-mod to initialize */
 
-	s->secmod_addr.sun_family = AF_UNIX;
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
-	p = s->socket_file;
+		s->sec_mod_instances[i].secmod_addr.sun_family = AF_UNIX;
-	if (GETPCONFIG(s)->chroot_dir) /* if we are on chroot make the socket file path relative */
+		p = s->sec_mod_instances[i].socket_file;
-		while (*p == '/') p++;
+		if (GETPCONFIG(s)->chroot_dir) /* if we are on chroot make the socket file path relative */
-	strlcpy(s->secmod_addr.sun_path, p, sizeof(s->secmod_addr.sun_path));
+			while (*p == '/') p++;
-	s->secmod_addr_len = SUN_LEN(&s->secmod_addr);
+		strlcpy(s->sec_mod_instances[i].secmod_addr.sun_path, p, sizeof(s->sec_mod_instances[i].secmod_addr.sun_path));
+		s->sec_mod_instances[i].secmod_addr_len = SUN_LEN(&s->sec_mod_instances[i].secmod_addr);
+	}
 
 	/* initialize memory for worker process */
 	worker_pool = talloc_named(main_pool, 0, "worker");
@@ -1504,7 +1555,10 @@ int main(int argc, char** argv)
 	ev_set_syserr_cb(syserr_cb);
 
 	ev_init(&ctl_watcher, ctl_watcher_cb);
-	ev_init(&sec_mod_watcher, sec_mod_watcher_cb);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		ev_init(&sec_mod_watchers[i].sec_mod_watcher, sec_mod_watcher_cb);
+		sec_mod_watchers[i].sec_mod_instance_index = i;
+	}
 
 	ev_init (&int_sig_watcher, term_sig_watcher_cb);
 	ev_signal_set (&int_sig_watcher, SIGINT);
@@ -1525,14 +1579,19 @@ int main(int argc, char** argv)
 		ev_io_start (loop, &ltmp->io);
 	}
 
-	ev_io_set(&sec_mod_watcher, s->sec_mod_fd, EV_READ);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		ev_io_set(&sec_mod_watchers[i].sec_mod_watcher, s->sec_mod_instances[i].sec_mod_fd, EV_READ);
+		ev_io_start (loop, &sec_mod_watchers[i].sec_mod_watcher);
+	}
+
 	ctl_handler_set_fds(s, &ctl_watcher);
 
 	ev_io_start (loop, &ctl_watcher);
-	ev_io_start (loop, &sec_mod_watcher);
 
-	ev_child_init(&child_watcher, sec_mod_child_watcher_cb, s->sec_mod_pid, 0);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
-	ev_child_start (loop, &child_watcher);
+		ev_child_init(&sec_mod_watchers[i].child_watcher, sec_mod_child_watcher_cb, s->sec_mod_instances[i].sec_mod_pid, 0);
+		ev_child_start (loop, &sec_mod_watchers[i].child_watcher);
+	}
 
 	ev_init(&maintenance_watcher, maintenance_watcher_cb);
 	ev_timer_set(&maintenance_watcher, MAIN_MAINTENANCE_TIME, MAIN_MAINTENANCE_TIME);
@@ -1555,7 +1614,9 @@ int main(int argc, char** argv)
 	/* try to clean-up everything allocated to ease checks 
 	 * for memory leaks.
 	 */
-	remove(s->full_socket_file);
+	for (i = 0; i < s->sec_mod_instance_count; i ++) {
+		remove(s->sec_mod_instances[i].full_socket_file);
+	}
 	remove(GETPCONFIG(s)->occtl_socket_file);
 	remove_pid_file();
 

src/main.h

@@ -58,6 +58,8 @@ int cmd_parser (void *pool, int argc, char **argv, struct list_head *head, bool
 #define LATENCY_AGGREGATION_TIME (60)
 #endif
 
+#define MINIMUM_USERS_PER_SEC_MOD 500
+
 struct listener_st {
 	ev_io io;
 	struct list_node list;
@@ -149,6 +151,7 @@ typedef struct proc_st {
 	char cstp_compr[8];
 	char dtls_compr[8];
 	unsigned mtu;
+	unsigned int sec_mod_instance_index;
 
 	/* if the session is initiated by a cookie the following two are set
 	 * and are considered when generating an IP address. That is used to
@@ -220,15 +223,9 @@ struct main_stats_st {
 	unsigned max_mtu;
 
 	unsigned active_clients;
-	/* updated on the cli_stats_msg from sec-mod. 
-	 * Holds the number of entries in secmod list of users */
-	unsigned secmod_client_entries;
-	unsigned tlsdb_entries;
 	time_t start_time;
 	time_t last_reset;
 
-	uint32_t avg_auth_time; /* in seconds */
-	uint32_t max_auth_time; /* in seconds */
 	uint32_t avg_session_mins; /* in minutes */
 	uint32_t max_session_mins;
 	uint64_t auth_failures; /* authentication failures */
@@ -243,6 +240,26 @@ struct main_stats_st {
 #endif
 };
 
+typedef struct sec_mod_instance_st {
+	struct main_server_st * server;
+	char socket_file[_POSIX_PATH_MAX];
+	char full_socket_file[_POSIX_PATH_MAX];
+	pid_t sec_mod_pid;
+
+	struct sockaddr_un secmod_addr;
+	unsigned secmod_addr_len;
+
+	int sec_mod_fd; /* messages are sent and received async */
+	int sec_mod_fd_sync; /* messages are send in a sync order (ping-pong). Only main sends. */
+	/* updated on the cli_stats_msg from sec-mod. 
+	 * Holds the number of entries in secmod list of users */
+	unsigned secmod_client_entries;
+	unsigned tlsdb_entries;
+	uint32_t avg_auth_time; /* in seconds */
+	uint32_t max_auth_time; /* in seconds */
+
+} sec_mod_instance_st;
+
 typedef struct main_server_st {
 	/* virtual hosts are only being added to that list, never removed */
 	struct list_head *vconfig;
@@ -256,13 +273,6 @@ typedef struct main_server_st {
 	struct script_list_st script_list;
 	/* maps DTLS session IDs to proc entries */
 	struct proc_hash_db_st proc_table;
-	
-	char socket_file[_POSIX_PATH_MAX];
-	char full_socket_file[_POSIX_PATH_MAX];
-	pid_t sec_mod_pid;
-
-	struct sockaddr_un secmod_addr;
-	unsigned secmod_addr_len;
 
 	struct main_stats_st stats;
 
@@ -271,11 +281,12 @@ typedef struct main_server_st {
 	/* This one is on worker pool */
 	struct worker_st *ws;
 
+	unsigned int sec_mod_instance_count;
+	sec_mod_instance_st * sec_mod_instances;
+
 	int top_fd;
 	int ctl_fd;
 
-	int sec_mod_fd; /* messages are sent and received async */
-	int sec_mod_fd_sync; /* messages are send in a sync order (ping-pong). Only main sends. */
 	void *main_pool; /* talloc main pool */
 	void *config_pool; /* talloc config pool */
 
@@ -292,7 +303,7 @@ typedef struct main_server_st {
 void clear_lists(main_server_st *s);
 
 int handle_worker_commands(main_server_st *s, struct proc_st* cur);
-int handle_sec_mod_commands(main_server_st *s);
+int handle_sec_mod_commands(sec_mod_instance_st * sec_mod_instances);
 
 int user_connected(main_server_st *s, struct proc_st* cur);
 void user_hostname_update(main_server_st *s, struct proc_st* cur);
@@ -300,8 +311,8 @@ void user_disconnected(main_server_st *s, struct proc_st* cur);
 
 int send_udp_fd(main_server_st* s, struct proc_st * proc, int fd);
 
-int session_open(main_server_st * s, struct proc_st *proc, const uint8_t *cookie, unsigned cookie_size);
+int session_open(sec_mod_instance_st * sec_mod_instance, struct proc_st *proc, const uint8_t *cookie, unsigned cookie_size);
-int session_close(main_server_st * s, struct proc_st *proc);
+int session_close(sec_mod_instance_st * sec_mod_instance, struct proc_st *proc);
 
 #ifdef UNDER_TEST
 /* for testing */
@@ -336,13 +347,13 @@ int set_tun_mtu(main_server_st* s, struct proc_st * proc, unsigned mtu);
 int send_cookie_auth_reply(main_server_st* s, struct proc_st* proc,
 			AUTHREP r);
 
-int handle_auth_cookie_req(main_server_st* s, struct proc_st* proc,
+int handle_auth_cookie_req(sec_mod_instance_st * sec_mod_instance, struct proc_st* proc,
  			   const AuthCookieRequestMsg * req);
 
 int check_multiple_users(main_server_st *s, struct proc_st* proc);
 int handle_script_exit(main_server_st *s, struct proc_st* proc, int code);
 
-int run_sec_mod(main_server_st * s, int *sync_fd);
+void run_sec_mod(sec_mod_instance_st * sec_mod_instance, unsigned int instance_index);
 
 struct proc_st *new_proc(main_server_st * s, pid_t pid, int cmd_fd,
 			struct sockaddr_storage *remote_addr, socklen_t remote_addr_len,
@@ -389,7 +400,7 @@ int send_socket_msg_to_worker(main_server_st* s, struct proc_st* proc, uint8_t c
 	return send_socket_msg(proc, proc->fd, cmd, socketfd, msg, get_size, pack);
 }
 
-int secmod_reload(main_server_st * s);
+int secmod_reload(sec_mod_instance_st * sec_mod_instance);
 
 const char *secmod_socket_file_name(struct perm_cfg_st *perm_config);
 void restore_secmod_socket_file_name(const char * save_path);

src/occtl/unix.c

@@ -234,7 +234,8 @@ int handle_status_cmd(struct unix_ctx *ctx, const char *arg, cmd_params_st *para
 
 		print_single_value(stdout, params, "Status", rep->status != 0 ? "online" : "error", 1);
 		print_single_value_int(stdout, params, "Server PID", rep->pid, 1);
-		print_single_value_int(stdout, params, "Sec-mod PID", rep->sec_mod_pid, 1);
+		print_single_value_int(stdout, params, "Sec-mod PID", rep->sec_mod_pids[0], 1);
+		print_single_value_int(stdout, params, "Sec-mod instance count", rep->n_sec_mod_pids, 1);
 
 		t = rep->start_time;
 		tm = localtime(&t);

src/sec-mod-db.c

@@ -110,6 +110,9 @@ client_entry_st *new_client_entry(sec_mod_st *sec, struct vhost_cfg_st *vhost, c
 			goto fail;
 		}
 
+		e->sid[0] = sec->sec_mod_instance_id;
+		seclog(sec, LOG_INFO, "sec-mod instance %d issue cookie", sec->sec_mod_instance_id);
+
 		/* check if in use */
 		te = find_client_entry(sec, e->sid);
 	} while(te != NULL && retries-- >= 0);

src/sec-mod.c

@@ -888,7 +888,8 @@ static int load_keys(sec_mod_st *sec, unsigned force)
  */
 void sec_mod_server(void *main_pool, void *config_pool, struct list_head *vconfig,
 		    const char *socket_file, int cmd_fd, int cmd_fd_sync,
-		    size_t  hmac_key_length, const uint8_t * hmac_key)
+		    size_t  hmac_key_length, const uint8_t * hmac_key,
+			const uint8_t instance_id)
 {
 	struct sockaddr_un sa;
 	socklen_t sa_len;
@@ -935,6 +936,7 @@ void sec_mod_server(void *main_pool, void *config_pool, struct list_head *vconfi
 	sec->config_pool = config_pool;
 	sec->sec_mod_pool = sec_mod_pool;
 	memcpy((uint8_t*)sec->hmac_key, hmac_key, hmac_key_length);
+	sec->sec_mod_instance_id = instance_id;
 
 	tls_cache_init(sec, &sec->tls_db);
 	sup_config_init(sec);

src/sec-mod.h

@@ -49,6 +49,7 @@ typedef struct sec_mod_st {
 	uint32_t total_authentications; /* successful authentications: to calculate the average above */
 	time_t last_stats_reset;
 	const uint8_t hmac_key[HMAC_DIGEST_SIZE];
+	uint32_t sec_mod_instance_id;
 } sec_mod_st;
 
 typedef struct stats_st {
@@ -169,6 +170,7 @@ void sec_auth_user_deinit(sec_mod_st *sec, client_entry_st *e);
 void sec_mod_server(void *main_pool, void *config_pool, struct list_head *vconfig,
 		    const char *socket_file,
 		    int cmd_fd, int cmd_fd_sync,
-			size_t  hmac_key_length, const uint8_t * hmac_key);
+			size_t  hmac_key_length, const uint8_t * hmac_key,
+			const uint8_t instance_id);
 
 #endif

src/vpn.h

@@ -397,6 +397,8 @@ struct perm_cfg_st {
 	unsigned int port;
 	unsigned int udp_port;
 
+	unsigned int sec_mod_scale;
+
 	/* for testing ocserv only */
 	unsigned debug_no_secmod_stats;
 

tests/Makefile.am

@@ -43,7 +43,7 @@ EXTRA_DIST = certs/ca-key.pem certs/ca.pem ns.sh common.sh certs/server-cert.pem
 	data/group-config/group1 data/test-namespace-listen.config data/disconnect-user.config \
 	data/disconnect-user2.config data/ping-leases.config data/haproxy-proxyproto.config \
 	data/haproxy-proxyproto.cfg scripts/proxy-connectscript data/haproxy-proxyproto-v1.config \
-	data/haproxy-proxyproto-v1.cfg scripts/proxy-connectscript-v1
+	data/haproxy-proxyproto-v1.cfg scripts/proxy-connectscript-v1 data/test-multiple-client-ip.config
 
 xfail_scripts =
 dist_check_SCRIPTS =  ocpasswd-test
@@ -71,7 +71,7 @@ if ENABLE_NUTTCP_TESTS
 dist_check_SCRIPTS += traffic lz4-compression lzs-compression \
 	aes256-cipher aes128-cipher oc-aes256-gcm-cipher oc-aes128-gcm-cipher \
 	test-config-per-group ac-aes128-gcm-cipher ac-aes256-gcm-cipher \
-	no-dtls-cipher psk-negotiate psk-negotiate-match
+	no-dtls-cipher psk-negotiate psk-negotiate-match test-multiple-client-ip
 endif
 
 if RADIUS_ENABLED

tests/data/test-multiple-client-ip.config

@@ -0,0 +1,188 @@
+# User authentication method. Could be set multiple times and in that case
+# all should succeed.
+# Options: certificate, pam. 
+#auth = "certificate"
+auth = "plain[@SRCDIR@/data/test1.passwd]"
+#auth = "pam"
+
+isolate-workers = @ISOLATE_WORKERS@
+
+max-ban-score = 0
+
+# A banner to be displayed on clients
+#banner = "Welcome"
+
+# Use listen-host to limit to specific IPs or to the IPs of a provided hostname.
+#listen-host = @ADDRESS@
+
+use-dbus = no
+
+# Limit the number of clients. Unset or set to zero for unlimited.
+#max-clients = 1024
+max-clients = 16
+
+listen-proxy-proto = false
+
+# Limit the number of client connections to one every X milliseconds 
+# (X is the provided value). Set to zero for no limit.
+#rate-limit-ms = 100
+
+# Limit the number of identical clients (i.e., users connecting multiple times)
+# Unset or set to zero for unlimited.
+max-same-clients = 2
+
+# TCP and UDP port number
+tcp-port = @PORT@
+udp-port = @PORT@
+
+# Keepalive in seconds
+keepalive = 32400
+
+# Dead peer detection in seconds
+dpd = 440
+
+# MTU discovery (DPD must be enabled)
+try-mtu-discovery = false
+
+# The key and the certificates of the server
+# The key may be a file, or any URL supported by GnuTLS (e.g., 
+# tpmkey:uuid=xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx;storage=user
+# or pkcs11:object=my-vpn-key;object-type=private)
+#
+# There may be multiple certificate and key pairs and each key
+# should correspond to the preceding certificate.
+server-cert = @SRCDIR@/certs/server-cert.pem
+server-key = @SRCDIR@/certs/server-key.pem
+
+# Diffie-Hellman parameters. Only needed if you require support
+# for the DHE ciphersuites (by default this server supports ECDHE).
+# Can be generated using:
+# certtool --generate-dh-params --outfile /path/to/dh.pem
+#dh-params = /path/to/dh.pem
+
+# If you have a certificate from a CA that provides an OCSP
+# service you may provide a fresh OCSP status response within
+# the TLS handshake. That will prevent the client from connecting
+# independently on the OCSP server.
+# You can update this response periodically using:
+# ocsptool --ask --load-cert=your_cert --load-issuer=your_ca --outfile response
+# Make sure that you replace the following file in an atomic way.
+#ocsp-response = /path/to/ocsp.der
+
+# In case PKCS #11 or TPM keys are used the PINs should be available
+# in files. The srk-pin-file is applicable to TPM keys only (It's the storage
+# root key).
+#pin-file = /path/to/pin.txt
+#srk-pin-file = /path/to/srkpin.txt
+
+# The Certificate Authority that will be used
+# to verify clients if certificate authentication
+# is set.
+#ca-cert = /path/to/ca.pem
+
+# The object identifier that will be used to read the user ID in the client certificate.
+# The object identifier should be part of the certificate's DN
+# Useful OIDs are: 
+#  CN = 2.5.4.3, UID = 0.9.2342.19200300.100.1.1
+#cert-user-oid = 0.9.2342.19200300.100.1.1
+
+# The object identifier that will be used to read the user group in the client 
+# certificate. The object identifier should be part of the certificate's DN
+# Useful OIDs are: 
+#  OU (organizational unit) = 2.5.4.11 
+#cert-group-oid = 2.5.4.11
+
+# A revocation list of ca-cert is set
+#crl = /path/to/crl.pem
+
+# GnuTLS priority string
+tls-priorities = "PERFORMANCE:%SERVER_PRECEDENCE:%COMPAT"
+
+# To enforce perfect forward secrecy (PFS) on the main channel.
+#tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA"
+
+# The time (in seconds) that a client is allowed to stay connected prior
+# to authentication
+auth-timeout = 40
+
+# The time (in seconds) that a client is not allowed to reconnect after 
+# a failed authentication attempt.
+#min-reauth-time = 2
+
+# Script to call when a client connects and obtains an IP
+# Parameters are passed on the environment.
+# REASON, USERNAME, GROUPNAME, HOSTNAME (the hostname selected by client), 
+# DEVICE, IP_REAL (the real IP of the client), IP_LOCAL (the local IP
+# in the P-t-P connection), IP_REMOTE (the VPN IP of the client). REASON
+# may be "connect" or "disconnect".
+#connect-script = /usr/bin/myscript
+#disconnect-script = /usr/bin/myscript
+
+# UTMP
+#use-utmp = true
+
+# PID file
+#pid-file = ./ocserv.pid
+
+# The default server directory. Does not require any devices present.
+#chroot-dir = /path/to/chroot
+
+# socket file used for IPC, will be appended with .PID
+# It must be accessible within the chroot environment (if any)
+socket-file = ./ocserv-socket
+
+occtl-socket-file = @OCCTL_SOCKET@
+use-occtl = true
+
+# The user the worker processes will be run as. It should be
+# unique (no other services run as this user).
+run-as-user = @USERNAME@
+run-as-group = @GROUP@
+
+# Network settings
+
+device = vpns
+
+# The default domain to be advertised
+default-domain = example.com
+
+ipv4-network = @VPNNET@
+# Use the keywork local to advertize the local P-t-P address as DNS server
+ipv4-dns = 192.168.1.1
+
+# The NBNS server (if any)
+#ipv4-nbns = 192.168.2.3
+
+ipv6-network = @VPNNET6@
+#address = 
+#ipv6-mask = 
+#ipv6-dns = 
+
+# Prior to leasing any IP from the pool ping it to verify that
+# it is not in use by another (unrelated to this server) host.
+ping-leases = false
+
+# Leave empty to assign the default MTU of the device
+# mtu = 
+
+#route = 192.168.1.0/255.255.255.0
+#route = 192.168.5.0/255.255.255.0
+
+#
+# The following options are for (experimental) AnyConnect client 
+# compatibility. They are only available if the server is built 
+# with --enable-anyconnect
+#
+
+# Client profile xml. A sample file exists in doc/profile.xml.
+# This file must be accessible from inside the worker's chroot. 
+# The profile is ignored by the openconnect client.
+#user-profile = profile.xml
+
+# Unless set to false it is required for clients to present their
+# certificate even if they are authenticating via a previously granted
+# cookie. Legacy CISCO clients do not do that, and thus this option
+# should be set for them.
+#always-require-cert = false
+
+sec-mod-scale = 2

tests/ns.sh

@@ -20,13 +20,16 @@
 
 # Input:
 #  ADDRESS=10.200.2.1
+#  ADDRESS2=10.200.2.2
 #  CLI_ADDRESS=10.200.1.1
+#  CLI_ADDRESS2=10.200.1.2
 #  VPNNET=192.168.1.0/24
 #  VPNADDR=192.168.1.1
 #
 # Provides:
 #  ${NSCMD1} - to run on NS1
 #  ${NSCMD2} - to run on NS2
+#  ${NSCMD3} - to run on NS3
 #
 # Cleanup is automatic via a trap
 #  Requires: finish() to be defined
@@ -55,44 +58,73 @@ function nsfinish {
   set +e
   test -n "${ETHNAME1}" && ${IP} link delete ${ETHNAME1} >/dev/null 2>&1
   test -n "${ETHNAME2}" && ${IP} link delete ${ETHNAME2} >/dev/null 2>&1
+  test -n "${ETHNAME3}" && ${IP} link delete ${ETHNAME3} >/dev/null 2>&1
+  test -n "${ETHNAME4}" && ${IP} link delete ${ETHNAME4} >/dev/null 2>&1
   test -n "${NSNAME1}" && ${IP} netns delete ${NSNAME1} >/dev/null 2>&1
   test -n "${NSNAME2}" && ${IP} netns delete ${NSNAME2} >/dev/null 2>&1
+  test -n "${NSNAME3}" && ${IP} netns delete ${NSNAME3} >/dev/null 2>&1
 
   finish
 }
 trap nsfinish EXIT
 
+# ETHNAME1 and ETHNAME2 are a veth pair
+# ETHNAME3 and ETHNAME4 are a veth pair
+# NSNAME1 and NSNAME3 are client namespaces containing ETHNAME1 and ETHNAME3
+# NSNAME2 is the server namespace containing ETHNAME2 and ETHNAME4
+
 echo " * Setting up namespaces..."
 set -e
 NSNAME1="ocserv-c-tmp-$$"
+NSNAME3="ocserv-c-2-tmp-$$"
 NSNAME2="ocserv-s-tmp-$$"
 ETHNAME1="oceth-c$$"
 ETHNAME2="oceth-s$$"
+ETHNAME3="oceth-c-2$$"
+ETHNAME4="oceth-s-2$$"
+
 ${IP} netns add ${NSNAME1}
 ${IP} netns add ${NSNAME2}
+${IP} netns add ${NSNAME3}
 
 ${IP} link add ${ETHNAME1} type veth peer name ${ETHNAME2}
 ${IP} link set ${ETHNAME1} netns ${NSNAME1}
 ${IP} link set ${ETHNAME2} netns ${NSNAME2}
 
+${IP} link add ${ETHNAME3} type veth peer name ${ETHNAME4}
+${IP} link set ${ETHNAME3} netns ${NSNAME3}
+${IP} link set ${ETHNAME4} netns ${NSNAME2}
+
 ${IP} -n ${NSNAME1} link set ${ETHNAME1} up
 ${IP} -n ${NSNAME2} link set ${ETHNAME2} up
+${IP} -n ${NSNAME3} link set ${ETHNAME3} up
+${IP} -n ${NSNAME2} link set ${ETHNAME4} up
 ${IP} -n ${NSNAME2} link set lo up
 
 ${IP} -n ${NSNAME1} addr add ${CLI_ADDRESS} dev ${ETHNAME1}
 ${IP} -n ${NSNAME2} addr add ${ADDRESS} dev ${ETHNAME2}
+test -n "${CLI_ADDRESS2}" && ${IP} -n ${NSNAME3} addr add ${CLI_ADDRESS2} dev ${ETHNAME3}
+test -n "${ADDRESS2}" && ${IP} -n ${NSNAME2} addr add ${ADDRESS2} dev ${ETHNAME4}
 
 ${IP} -n ${NSNAME1} route add default via ${CLI_ADDRESS} dev ${ETHNAME1}
+${IP} -n ${NSNAME2} route
 ${IP} -n ${NSNAME2} route add default via ${ADDRESS} dev ${ETHNAME2}
 
+test -n "${CLI_ADDRESS2}" && ${IP} -n ${NSNAME3} route add default via ${CLI_ADDRESS2} dev ${ETHNAME3}
+test -n "${ADDRESS2}" && ${IP} -n ${NSNAME2} route add ${CLI_ADDRESS2}/32 via ${ADDRESS2} dev ${ETHNAME4}
+
 ${IP} -n ${NSNAME2} addr
 ${IP} -n ${NSNAME2} route
 ${IP} -n ${NSNAME1} route
+${IP} -n ${NSNAME3} route
 
-${IP} netns exec ${NSNAME1} ping -c 1 ${ADDRESS} >/dev/null
+${IP} netns exec ${NSNAME1} ping -c 1 ${ADDRESS} >/dev/null 
-${IP} netns exec ${NSNAME2} ping -c 1 ${ADDRESS} >/dev/null
+${IP} netns exec ${NSNAME2} ping -c 1 ${ADDRESS} >/dev/null 
 ${IP} netns exec ${NSNAME2} ping -c 1 ${CLI_ADDRESS} >/dev/null
+test -n "${ADDRESS2}" && ${IP} netns exec ${NSNAME2} ping -c 1 ${ADDRESS2} >/dev/null 
+test -n "${CLI_ADDRESS2}" && ${IP} netns exec ${NSNAME2} ping -c 1 ${CLI_ADDRESS2} >/dev/null 
 set +e
 
 CMDNS1="${IP} netns exec ${NSNAME1}"
 CMDNS2="${IP} netns exec ${NSNAME2}"
+CMDNS3="${IP} netns exec ${NSNAME3}"

tests/test-multiple-client-ip

@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 Microsoft Corporation
+#
+# This file is part of ocserv.
+#
+# ocserv is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at
+# your option) any later version.
+#
+# ocserv is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+OCCTL="${OCCTL:-../src/occtl/occtl}"
+SERV="${SERV:-../src/ocserv}"
+srcdir=${srcdir:-.}
+PIDFILE=ocserv-pid.$$.tmp
+CLIPID=oc-pid.$$.tmp
+CLIPID2=oc-pid.2.$$.tmp
+PATH=${PATH}:/usr/sbin
+IP=$(which ip)
+OUTFILE=traffic.$$.tmp
+
+. `dirname $0`/common.sh
+
+eval "${GETPORT}"
+
+if test -z "${IP}";then
+	echo "no IP tool is present"
+	exit 77
+fi
+
+if test "$(id -u)" != "0";then
+	echo "This test must be run as root"
+	exit 77
+fi
+
+echo "Testing ocserv connection via haproxy... "
+
+function finish {
+  set +e
+  echo " * Cleaning up..."
+  test -n "${PID}" && kill ${PID} >/dev/null 2>&1
+  test -n "${PIDFILE}" && rm -f ${PIDFILE} >/dev/null 2>&1
+  test -n "${CLIPID}" && kill $(cat ${CLIPID}) >/dev/null 2>&1
+  test -n "${CLIPID}" && rm -f ${CLIPID} >/dev/null 2>&1
+  test -n "${CLIPID2}" && kill $(cat ${CLIPID2}) >/dev/null 2>&1
+  test -n "${CLIPID2}" && rm -f ${CLIPID2} >/dev/null 2>&1
+  test -n "${CONFIG}" && rm -f ${CONFIG} >/dev/null 2>&1
+  rm -f ${OUTFILE} 2>&1
+}
+trap finish EXIT
+
+# server address
+ADDRESS=10.200.2.1
+ADDRESS2=10.200.2.2
+CLI_ADDRESS=10.200.1.1
+CLI_ADDRESS2=10.200.1.2
+VPNNET=192.168.1.0/24
+VPNADDR=192.168.1.1
+VPNNET6=fd91:6d87:7341:db6a::/112
+VPNADDR6=fd91:6d87:7341:db6a::1
+OCCTL_SOCKET=./occtl-traffic-$$.socket
+USERNAME=test
+
+. `dirname $0`/ns.sh
+
+# Run servers
+update_config test-multiple-client-ip.config
+if test "$VERBOSE" = 1;then
+DEBUG="-d 3"
+fi
+
+${CMDNS2} ${SERV} -p ${PIDFILE} -f -c ${CONFIG} ${DEBUG} & PID=$!
+
+sleep 4
+
+# Run client 1
+echo " * Getting cookie from ${ADDRESS}:${PORT}..."
+( echo "test" | ${CMDNS1} ${OPENCONNECT} ${ADDRESS}:${PORT} -u ${USERNAME} --servercert=d66b507ae074d03b02eafca40d35f87dd81049d3 --cookieonly )
+if test $? != 0;then
+	echo "Could not get cookie from server"
+	exit 1
+fi
+
+echo " * Connecting to ${ADDRESS}:${PORT}..."
+( echo "test" | ${CMDNS1} ${OPENCONNECT} ${ADDRESS}:${PORT} -u ${USERNAME} --servercert=d66b507ae074d03b02eafca40d35f87dd81049d3 -s ${srcdir}/scripts/vpnc-script --pid-file=${CLIPID} --passwd-on-stdin -b )
+if test $? != 0;then
+	echo "Could not connect to server"
+	exit 1
+fi
+
+# Run client 2
+echo " * Getting cookie from ${ADDRESS}:${PORT}..."
+( echo "test" | ${CMDNS3} ${OPENCONNECT} ${ADDRESS}:${PORT} -u ${USERNAME} --servercert=d66b507ae074d03b02eafca40d35f87dd81049d3 --cookieonly )
+if test $? != 0;then
+	echo "Could not get cookie from server"
+	exit 1
+fi
+
+echo " * Connecting to ${ADDRESS}:${PORT}..."
+( echo "test" | ${CMDNS3} ${OPENCONNECT} ${ADDRESS}:${PORT} -u ${USERNAME} --servercert=d66b507ae074d03b02eafca40d35f87dd81049d3 -s ${srcdir}/scripts/vpnc-script --pid-file=${CLIPID2} --passwd-on-stdin -b )
+if test $? != 0;then
+	echo "Could not connect to server"
+	exit 1
+fi
+
+${OCCTL} -s ${OCCTL_SOCKET} show users|grep ${USERNAME}
+if test $? != 0;then
+	echo "occtl didn't find connected user!"
+	exit 1
+fi
+
+${OCCTL} -s ${OCCTL_SOCKET} show user ${USERNAME} >${OUTFILE}
+if test $? != 0;then
+	echo "occtl didn't find connected user!"
+	exit 1
+fi
+
+grep "Username: ${USERNAME}" ${OUTFILE}
+if test $? != 0;then
+	echo "occtl show user didn't find connected user!"
+	exit 1
+fi
+
+grep ${CLI_ADDRESS} ${OUTFILE}
+if test $? != 0;then
+	echo "occtl show user didn't find client address!"
+	exit 1
+fi
+
+grep ${CLI_ADDRESS2} ${OUTFILE}
+if test $? != 0;then
+	echo "occtl show user didn't find client address2!"
+	exit 1
+fi
+
+${OCCTL} -s ${OCCTL_SOCKET} show sessions valid >${OUTFILE}
+
+grep ${CLI_ADDRESS} ${OUTFILE}
+if test $? != 0;then
+	echo "occtl show sessions didn't find client address!"
+	exit 1
+fi
+
+grep ${CLI_ADDRESS2} ${OUTFILE}
+if test $? != 0;then
+	echo "occtl show sessions didn't find client address2!"
+	exit 1
+fi
+
+
+exit 0