pam: increase coroutine stack to 8 MB and add guard page

Certain pam modules such as pam_sss with AD/Kerberos and multi-factor
authentication requires significantly more stack than the previous
1 MB limit.

On Linux, allocate the coroutine stack with mmap and place a PROT_NONE
guard page immediately below it.  This turns a stack overflow into an
immediate SIGSEGV rather than silent corruption of adjacent heap memory
to better detect similar cases.

Fixes: #657
Relates: #619

Signed-off-by: Nikos Mavrogiannopoulos <n.mavrogiannopoulos@gmail.com>

NEWS

@@ -11,6 +11,8 @@
 - Distinguish disconnect reasons in AnyConnect BYE packets (#732)
 - Vhosts now inherit configuration options from the default vhost if
   they are not overridden (#705)
+- Improved PAM handling by raising the provided stack memory to the default
+  for the system, and better detect overflow (#657)
 - `tunnel-all-dns` now works correctly when set in per-user/group config (#708)
 - radius: fixed Framed-IPv6-Prefix routes being silently dropped (#710)
 - Added per-worker memory limit (RLIMIT_DATA) as defense-in-depth against

src/auth/pam-stack.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2026 Nikos Mavrogiannopoulos
+ *
+ * This file is part of ocserv.
+ *
+ * ocserv is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * ocserv is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef PAM_STACK_H
+#define PAM_STACK_H
+
+#include <config.h>
+#include <stdlib.h>
+#ifdef __linux__
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#endif
+
+/* Minimum coroutine stack; lower values caused silent heap corruption
+ * (#619, #657).  The actual size is read from RLIMIT_STACK at runtime. */
+#define MIN_PAM_STACK_SIZE (8 * 1024 * 1024)
+
+struct pam_stack_st {
+	void *base; /* mmap base (Linux) or malloc'd pointer */
+#ifdef __linux__
+	size_t total; /* total mmap size: guard page + usable area */
+#endif
+};
+
+/* Returns the coroutine stack size: RLIMIT_STACK if it is finite and at
+ * least MIN_PAM_STACK_SIZE, otherwise MIN_PAM_STACK_SIZE. */
+static inline size_t pam_stack_size(void)
+{
+#ifdef __linux__
+	struct rlimit rl;
+	if (getrlimit(RLIMIT_STACK, &rl) == 0 && rl.rlim_cur != RLIM_INFINITY &&
+	    rl.rlim_cur >= MIN_PAM_STACK_SIZE)
+		return (size_t)rl.rlim_cur;
+#endif
+	return MIN_PAM_STACK_SIZE;
+}
+
+#ifdef __linux__
+/* Returns the usable stack base to pass to co_create, storing the full mmap
+ * region in st for cleanup.  Returns NULL on failure (PCL falls back to
+ * malloc). */
+static inline void *pam_stack_alloc(struct pam_stack_st *st, size_t size)
+{
+	long pgsz = sysconf(_SC_PAGESIZE);
+	size_t total;
+	void *map;
+
+	if (pgsz <= 0)
+		return NULL;
+	total = (size_t)pgsz + size;
+	map = mmap(NULL, total, PROT_READ | PROT_WRITE,
+		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+	if (map == MAP_FAILED)
+		return NULL;
+	/* Guard page below the stack: overflow → SIGSEGV, not silent heap corruption. */
+	if (mprotect(map, (size_t)pgsz, PROT_NONE) != 0) {
+		munmap(map, total);
+		return NULL;
+	}
+	st->base = map;
+	st->total = total;
+	return (char *)map + pgsz;
+}
+
+static inline void pam_stack_free(struct pam_stack_st *st)
+{
+	if (st->base != NULL) {
+		munmap(st->base, st->total);
+		st->base = NULL;
+	}
+}
+#else /* !__linux__ */
+static inline void *pam_stack_alloc(struct pam_stack_st *st, size_t size)
+{
+	st->base = malloc(size);
+	return st->base;
+}
+static inline void pam_stack_free(struct pam_stack_st *st)
+{
+	free(st->base);
+	st->base = NULL;
+}
+#endif /* __linux__ */
+
+#endif /* PAM_STACK_H */

src/auth/pam.c

@@ -48,8 +48,6 @@
 #include "auth/pam.h"
 #include "auth-unix.h"
 
-#define PAM_STACK_SIZE (1024 * 1024)
-
 #define MAX_REPLIES 2
 
 enum {
@@ -235,9 +233,14 @@ static int pam_auth_init(void **ctx, void *pool, void *vctx,
 		goto fail1;
 	}
 
-	pctx->cr = co_create(co_auth_user, pctx, NULL, PAM_STACK_SIZE);
-	if (pctx->cr == NULL)
+	size_t ssize = pam_stack_size();
+	pctx->cr = co_create(co_auth_user, pctx,
+			     pam_stack_alloc(&pctx->cr_stack, ssize),
+			     (int)ssize);
+	if (pctx->cr == NULL) {
+		pam_stack_free(&pctx->cr_stack);
 		goto fail2;
+	}
 
 	strlcpy(pctx->username, info->username, sizeof(pctx->username));
 
@@ -380,6 +383,7 @@ static void pam_auth_deinit(void *ctx)
 	str_clear(&pctx->msg);
 	if (pctx->cr != NULL)
 		co_delete(pctx->cr);
+	pam_stack_free(&pctx->cr_stack);
 	talloc_free(pctx);
 }
 

src/auth/pam.h

@@ -29,6 +29,7 @@
 #include <security/pam_appl.h>
 #include <str.h>
 #include <pcl.h>
+#include "pam-stack.h"
 
 extern const struct auth_mod_st pam_auth_funcs;
 
@@ -38,6 +39,7 @@ struct pam_ctx_st {
 	pam_handle_t *ph;
 	struct pam_conv dc;
 	coroutine_t cr;
+	struct pam_stack_st cr_stack;
 	int cr_ret;
 	unsigned int changing; /* whether we are entering a new password */
 	str_st msg;

tests/meson.build

@@ -79,6 +79,18 @@ config_inherit_exe = executable('config-inherit', 'config-inherit.c',
 test('config-inherit', config_inherit_exe,
   env: test_env, workdir: test_workdir, timeout: 30)
 
+# pam-guard-page: verifies that the PROT_NONE guard page beneath the PCL
+# coroutine stack converts a stack overflow into a crash (#619, #657).
+# Linux only; returns 77 (skip) on other platforms.
+if pam_dep.found()
+  pam_guard_page_exe = executable('pam-guard-page', 'pam-guard-page.c',
+    dependencies: test_base_deps + [pcl_dep],
+    include_directories: test_inc,
+  )
+  test('pam-guard-page', pam_guard_page_exe,
+    env: test_env, workdir: test_workdir, timeout: 30)
+endif
+
 # gen_oidc_test_data (only when OIDC enabled)
 if oidc_enabled
   gen_oidc_exe = executable('gen_oidc_test_data', 'generate_oidc_test_data.c',

tests/pam-guard-page.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2026 Nikos Mavrogiannopoulos
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Verify that pam_stack_alloc() produces a PROT_NONE guard page that converts
+ * coroutine stack overflow into a deterministic fault rather than silent heap
+ * corruption (issues #619, #657).
+ *
+ * A PCL coroutine is created on the stack returned by pam_stack_alloc().  The
+ * coroutine recurses until the stack is exhausted.  A SIGSEGV handler (running
+ * on an alternate signal stack) checks that the fault address falls within the
+ * guard page — proving it was the guard page that fired, not an accidental
+ * fault past the end of an unguarded malloc'd buffer.
+ *
+ * Linux only — pam_stack_alloc() installs a guard page only on Linux.
+ * Returns 77 (meson skip) on other platforms.
+ */
+
+#include <config.h>
+
+#ifndef __linux__
+int main(void)
+{
+	return 77; /* skip on non-Linux */
+}
+#else
+
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <pcl.h>
+#include "auth/pam-stack.h"
+
+/* Small coroutine stack: enough for PCL bookkeeping, small enough to overflow
+ * quickly under the recursive load below. */
+#define TEST_STACK_SIZE (64 * 1024)
+
+/* Alternate signal stack — static so no allocation is needed after fork. */
+static char altstack_buf[65536];
+
+/* Guard page extent, set before fork so the child inherits the values. */
+static void *guard_base;
+static size_t guard_size;
+
+static void sigsegv_handler(int sig, siginfo_t *si, void *ctx)
+{
+	(void)sig;
+	(void)ctx;
+	/* _exit(0) only when the fault is precisely in the guard page. */
+	if ((char *)si->si_addr >= (char *)guard_base &&
+	    (char *)si->si_addr < (char *)guard_base + guard_size)
+		_exit(0);
+	/* Fault elsewhere — not the guard page; report failure. */
+	_exit(1);
+}
+
+static void recurse(int depth);
+/* volatile pointer breaks static infinite-recursion analysis */
+static void (*volatile recurse_ptr)(int) = recurse;
+
+/* Each frame consumes ~512 bytes; ~128 frames exhaust a 64 KB stack. */
+static void recurse(int depth)
+{
+	volatile char frame[512];
+	frame[0] = (char)depth;
+	(void)frame[0]; /* read back to prevent the frame being optimized away */
+	recurse_ptr(depth + 1);
+}
+
+static void overflow_coroutine(void *data)
+{
+	(void)data;
+	recurse(0);
+	/* unreachable — loop so PCL never sees a clean return */
+	while (1)
+		co_resume();
+}
+
+int main(void)
+{
+	struct pam_stack_st st = { 0 };
+	struct sigaction sa;
+	stack_t ss;
+	void *stack;
+	pid_t pid;
+	int status;
+
+	stack = pam_stack_alloc(&st, TEST_STACK_SIZE);
+	if (stack == NULL) {
+		fprintf(stderr, "pam_stack_alloc failed\n");
+		return 77;
+	}
+
+	/* Record the guard page extent for the signal handler. */
+	guard_base = st.base;
+	guard_size = (size_t)((char *)stack - (char *)st.base);
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fork");
+		pam_stack_free(&st);
+		return 1;
+	}
+
+	if (pid == 0) {
+		/* Child: install an alternate signal stack so the SIGSEGV
+		 * handler can run even after the coroutine stack is exhausted. */
+		ss.ss_sp = altstack_buf;
+		ss.ss_size = sizeof(altstack_buf);
+		ss.ss_flags = 0;
+		if (sigaltstack(&ss, NULL) == -1) {
+			perror("sigaltstack");
+			_exit(1);
+		}
+
+		sa.sa_sigaction = sigsegv_handler;
+		sigemptyset(&sa.sa_mask);
+		sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
+		if (sigaction(SIGSEGV, &sa, NULL) == -1) {
+			perror("sigaction");
+			_exit(1);
+		}
+
+		coroutine_t cr = co_create(overflow_coroutine, NULL, stack,
+					   TEST_STACK_SIZE);
+		if (cr == NULL) {
+			fprintf(stderr, "co_create failed\n");
+			_exit(1);
+		}
+		co_call(cr);
+		/* Reached only if the coroutine returned without overflowing. */
+		_exit(1);
+	}
+
+	/* Parent owns the mapping; child has its own copy after fork. */
+	pam_stack_free(&st);
+
+	if (waitpid(pid, &status, 0) < 0) {
+		perror("waitpid");
+		return 1;
+	}
+
+	if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
+		fprintf(stderr,
+			"PASS: fault address confirmed in guard page\n");
+		return 0;
+	}
+
+	if (WIFEXITED(status))
+		fprintf(stderr,
+			"FAIL: child exited with status %d "
+			"(fault outside guard page or no fault)\n",
+			WEXITSTATUS(status));
+	else
+		fprintf(stderr,
+			"FAIL: child killed by signal %d "
+			"(SIGSEGV handler did not run?)\n",
+			WTERMSIG(status));
+	return 1;
+}
+
+#endif /* __linux__ */