From 22a583d5d4a98d1b5183823dcd10f9a87e65eb47 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 30 Jul 2026 23:42:27 +0000 Subject: [PATCH 1/9] Minimize window logging thread holds logging_thread_mutex --- SymCryptProvider/src/p_scossl_keysinuse.c | 56 +++++++++++++---------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 58945a7a..0742fad1 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -837,7 +837,7 @@ static void p_scossl_keysinuse_log_common(int level, const char *message, va_lis int fd; for (int i = 0; i < 3; i++) { - fd = open(log_path, O_WRONLY | O_APPEND | O_CREAT, 0200); + fd = open(log_path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0200); if (fd >= 0 || errno != EACCES) { break; @@ -1025,34 +1025,44 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) p_scossl_keysinuse_log_error("Failed to lock keysinuse info stack,OPENSSL_%d", ERR_get_error()); } - // Log all pending usage events under lock. We need to lock in this section - // in case fork is called - if ((pthreadErr = pthread_mutex_lock(&logging_thread_mutex)) == 0) + // Log all pending usage events. logging_thread_mutex is held only around + // during the pKeysinuseInfo update to ensure the logging thread is not + // holding a lock during a fork. + while (sk_SCOSSL_PROV_KEYSINUSE_INFO_num(sk_keysinuse_info_pending) > 0) { - while (sk_SCOSSL_PROV_KEYSINUSE_INFO_num(sk_keysinuse_info_pending) > 0) + if ((pthreadErr = pthread_mutex_lock(&logging_thread_mutex)) != 0) { - pKeysinuseInfo = sk_SCOSSL_PROV_KEYSINUSE_INFO_pop(sk_keysinuse_info_pending); - if (CRYPTO_THREAD_write_lock(pKeysinuseInfo->lock)) - { - now = time(NULL); + p_scossl_keysinuse_log_error("Logging thread failed to accquire mutex,SYS_%d", pthreadErr); + goto cleanup; + } - pKeysinuseInfo->firstLogTime = pKeysinuseInfo->lastLogTime == 0 ? now : pKeysinuseInfo->firstLogTime; - pKeysinuseInfo->lastLogTime = now; - pKeysinuseInfo->logPending = FALSE; + pKeysinuseInfo = sk_SCOSSL_PROV_KEYSINUSE_INFO_pop(sk_keysinuse_info_pending); + if (pKeysinuseInfo != NULL && + CRYPTO_THREAD_write_lock(pKeysinuseInfo->lock)) + { + now = time(NULL); - keysinuseInfoTmp = *pKeysinuseInfo; + pKeysinuseInfo->firstLogTime = pKeysinuseInfo->lastLogTime == 0 ? now : pKeysinuseInfo->firstLogTime; + pKeysinuseInfo->lastLogTime = now; + pKeysinuseInfo->logPending = FALSE; - pKeysinuseInfo->decryptCounter = 0; - pKeysinuseInfo->signCounter = 0; + keysinuseInfoTmp = *pKeysinuseInfo; - CRYPTO_THREAD_unlock(pKeysinuseInfo->lock); - } - else - { - p_scossl_keysinuse_log_error("Failed to lock keysinuse info,OPENSSL_%d", ERR_get_error()); - keysinuseInfoTmp.refCount = -1; - } + pKeysinuseInfo->decryptCounter = 0; + pKeysinuseInfo->signCounter = 0; + + CRYPTO_THREAD_unlock(pKeysinuseInfo->lock); + } + else + { + p_scossl_keysinuse_log_error("Failed to lock keysinuse info,OPENSSL_%d", ERR_get_error()); + keysinuseInfoTmp.refCount = -1; + } + pthread_mutex_unlock(&logging_thread_mutex); + + if (pKeysinuseInfo != NULL) + { p_scossl_keysinuse_info_free(pKeysinuseInfo); if (keysinuseInfoTmp.refCount > 0) @@ -1065,8 +1075,6 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) keysinuseInfoTmp.lastLogTime); } } - - pthread_mutex_unlock(&logging_thread_mutex); } } while (isLoggingThreadRunning); From 0df8c9cef205c01da6e1d1946045f381e0dd7da8 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 6 Aug 2026 22:52:45 +0000 Subject: [PATCH 2/9] Remove unecessary null check --- SymCryptProvider/src/p_scossl_keysinuse.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 0742fad1..ee4cfa63 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -1061,19 +1061,16 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) pthread_mutex_unlock(&logging_thread_mutex); - if (pKeysinuseInfo != NULL) - { - p_scossl_keysinuse_info_free(pKeysinuseInfo); + p_scossl_keysinuse_info_free(pKeysinuseInfo); - if (keysinuseInfoTmp.refCount > 0) - { - p_scossl_keysinuse_log_notice("%s,%d,%d,%ld,%ld", - keysinuseInfoTmp.keyIdentifier, - keysinuseInfoTmp.signCounter, - keysinuseInfoTmp.decryptCounter, - keysinuseInfoTmp.firstLogTime, - keysinuseInfoTmp.lastLogTime); - } + if (keysinuseInfoTmp.refCount > 0) + { + p_scossl_keysinuse_log_notice("%s,%d,%d,%ld,%ld", + keysinuseInfoTmp.keyIdentifier, + keysinuseInfoTmp.signCounter, + keysinuseInfoTmp.decryptCounter, + keysinuseInfoTmp.firstLogTime, + keysinuseInfoTmp.lastLogTime); } } } From 86f06b7c33a3ffe6d855c1cccbbf56169673bf8d Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Tue, 11 Aug 2026 18:46:12 +0000 Subject: [PATCH 3/9] Don't consume signals from logging thread Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e08c0717-91d1-47dc-abd8-e5a8d0ecf076 --- SymCryptProvider/src/p_scossl_keysinuse.c | 42 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index ee4cfa63..6680768b 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,8 @@ static void p_scossl_keysinuse_init_once() int pthreadErr; SCOSSL_STATUS status = SCOSSL_FAILURE; BOOL attr_initialized = FALSE; + sigset_t oldSigSet; + sigset_t blockSigSet; // Store process PID for later use pid = getpid(); @@ -207,8 +210,23 @@ static void p_scossl_keysinuse_init_once() is_logging = TRUE; if ((pthreadErr = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) != 0 || - (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0 || - (pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL)) != 0) + (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0) + { + p_scossl_keysinuse_log_error("Failed to initialize logging thread condition,SYS_%d", pthreadErr); + is_logging = FALSE; + goto cleanup; + } + + // Block all signals across creation of the logging thread so it inherits a + // fully-blocked mask and never consumes signals intended for other threads + // (for example SIGIO, which the NGINX master relies on for worker channel + // acknowledgements). The previous mask is restored immediately afterwards. + sigfillset(&blockSigSet); + pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); + pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); + pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); + + if (pthreadErr != 0) { p_scossl_keysinuse_log_error("Failed to start logging thread,SYS_%d", pthreadErr); is_logging = FALSE; @@ -335,6 +353,8 @@ static void p_scossl_keysinuse_child() SCOSSL_STATUS status = SCOSSL_FAILURE; int is_parent_logging = is_logging; BOOL attr_initialized = FALSE; + sigset_t oldSigSet; + sigset_t blockSigSet; if (!keysinuse_enabled) { @@ -411,8 +431,22 @@ static void p_scossl_keysinuse_child() is_logging = TRUE; if ((pthreadErr = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) != 0 || - (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0 || - (pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL)) != 0) + (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0) + { + p_scossl_keysinuse_log_error("Failed to initialize logging thread condition,SYS_%d", pthreadErr); + is_logging = FALSE; + goto cleanup; + } + + // Block all signals across creation of the logging thread so it + // inherits a fully-blocked mask and never consumes signals intended + // for other threads. The previous mask is restored afterwards. + sigfillset(&blockSigSet); + pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); + pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); + pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); + + if (pthreadErr != 0) { p_scossl_keysinuse_log_error("Failed to start logging thread,SYS_%d", pthreadErr); is_logging = FALSE; From 02749634f7d459063f5f665f5eb2a0bb6601df44 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 20 Aug 2026 21:42:29 +0000 Subject: [PATCH 4/9] Add config to tweak keysinuse forking behavior --- SymCryptProvider/src/p_scossl_base.c | 26 +++ SymCryptProvider/src/p_scossl_keysinuse.c | 206 ++++++++++------------ SymCryptProvider/src/p_scossl_keysinuse.h | 6 + 3 files changed, 130 insertions(+), 108 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_base.c b/SymCryptProvider/src/p_scossl_base.c index 10a52482..0b67ec97 100644 --- a/SymCryptProvider/src/p_scossl_base.c +++ b/SymCryptProvider/src/p_scossl_base.c @@ -24,6 +24,7 @@ extern "C" { #define CONF_KEYSINUSE_ENABLED "keysinuse.enabled" #define CONF_KEYSINUSE_MAX_FILE_SIZE "keysinuse.max_file_size" #define CONF_KEYSINUSE_LOGGING_DELAY "keysinuse.logging_delay_seconds" +#define CONF_KEYSINUSE_PROCESS_SCOPE "keysinuse.process_scope" // Cap configured file size at 2GB #define SCOSSL_MAX_CONFIGURABLE_FILE_SIZE (2l << 30) @@ -413,12 +414,15 @@ static void p_scossl_start_keysinuse(_In_ const OSSL_CORE_HANDLE *handle) const char *confEnabled = NULL; const char *confMaxFileSize = NULL; const char *confLoggingDelay = NULL; + const char *confProcessScope = NULL; + const char *envProcessScope = NULL; const char *envEnabled = NULL; OSSL_PARAM keysinuseParams[] = { OSSL_PARAM_utf8_ptr(CONF_KEYSINUSE_ENABLED, &confEnabled, 0), OSSL_PARAM_utf8_ptr(CONF_KEYSINUSE_MAX_FILE_SIZE, &confMaxFileSize, 0), OSSL_PARAM_utf8_ptr(CONF_KEYSINUSE_LOGGING_DELAY, &confLoggingDelay, 0), + OSSL_PARAM_utf8_ptr(CONF_KEYSINUSE_PROCESS_SCOPE, &confProcessScope, 0), OSSL_PARAM_END}; // Config related errors shouldn't surface to caller @@ -501,6 +505,28 @@ static void p_scossl_start_keysinuse(_In_ const OSSL_CORE_HANDLE *handle) p_scossl_keysinuse_set_logging_delay(atol(confLoggingDelay)); } + // Environment overrides config. Config value is alreday fetched core_get_params above + if ((envProcessScope = NCONF_get_string(NULL, NULL, "KEYSINUSE_PROCESS_SCOPE")) != NULL) + { + confProcessScope = envProcessScope; + } + + if (confProcessScope != NULL) + { + if (OPENSSL_strcasecmp(confProcessScope, "main") == 0) + { + p_scossl_keysinuse_set_process_scope(KEYSINUSE_PROCESS_SCOPE_MAIN); + } + else if (OPENSSL_strcasecmp(confProcessScope, "child") == 0) + { + p_scossl_keysinuse_set_process_scope(KEYSINUSE_PROCESS_SCOPE_CHILD); + } + else + { + p_scossl_keysinuse_set_process_scope(KEYSINUSE_PROCESS_SCOPE_BOTH); + } + } + p_scossl_keysinuse_init(); } diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 6680768b..15a1a9a9 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -26,6 +26,8 @@ static off_t max_file_size = 5 << 10; // Default to 5KB static long logging_delay = 60 * 60; // Default to 1 hour static BOOL keysinuse_enabled = FALSE; static BOOL p_scossl_keysinuse_child_enabled = FALSE; +static BOOL keysinuse_initialized = FALSE; +static UINT32 keysinuse_process_scope = KEYSINUSE_PROCESS_SCOPE_BOTH; static pid_t pid = 0; static pid_t logging_thread_tid = 0; @@ -108,6 +110,72 @@ static void p_scossl_keysinuse_logging_thread_cleanup() } } +// Starts the logging thread and marks keysinuse as enabled. Reainitializes +// lock thread globals and state. On failure caller is repsonsible for cleanup. +static SCOSSL_STATUS p_scossl_keysinuse_create_logging_thread() +{ + pthread_condattr_t attr; + int pthreadErr; + BOOL attr_initialized = FALSE; + sigset_t oldSigSet; + sigset_t blockSigSet; + SCOSSL_STATUS status = SCOSSL_FAILURE; + + // Monotonic clock needs to be set to prevent wall clock changes from + // affecting the logging delay sleep time. Allocate condition variable, + // releasing any instance inherited across a fork first. + OPENSSL_free(logging_thread_cond_wake_early); + logging_thread_cond_wake_early = NULL; + + if ((logging_thread_cond_wake_early = OPENSSL_malloc(sizeof(pthread_cond_t))) == NULL) + { + p_scossl_keysinuse_log_error("Failed to allocate condition variable"); + goto cleanup; + } + + if ((pthreadErr = pthread_condattr_init(&attr)) != 0) + { + p_scossl_keysinuse_log_error("Failed to init condition attributes,SYS_%d", pthreadErr); + goto cleanup; + } + + attr_initialized = TRUE; + is_logging = TRUE; + + if ((pthreadErr = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) != 0 || + (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0) + { + p_scossl_keysinuse_log_error("Failed to initialize logging thread condition,SYS_%d", pthreadErr); + is_logging = FALSE; + goto cleanup; + } + + // Block all signals across creation of the logging thread so signal handlers + // never run in the logging thread. + sigfillset(&blockSigSet); + pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); + pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); + pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); + + if (pthreadErr != 0) + { + p_scossl_keysinuse_log_error("Failed to start logging thread,SYS_%d", pthreadErr); + is_logging = FALSE; + goto cleanup; + } + + keysinuse_enabled = TRUE; + status = SCOSSL_SUCCESS; + +cleanup: + if (attr_initialized) + { + pthread_condattr_destroy(&attr); + } + + return status; +} + static void p_scossl_keysinuse_init_once() { int mkdirResult; @@ -118,12 +186,8 @@ static void p_scossl_keysinuse_init_once() char *procPath = NULL; int cbProcPath = PATH_MAX; int cbProcPathUsed = 0; - pthread_condattr_t attr; int pthreadErr; SCOSSL_STATUS status = SCOSSL_FAILURE; - BOOL attr_initialized = FALSE; - sigset_t oldSigSet; - sigset_t blockSigSet; // Store process PID for later use pid = getpid(); @@ -191,48 +255,6 @@ static void p_scossl_keysinuse_init_once() goto cleanup; } - // Start the logging thread. Monotonic clock needs to be set to - // prevent wall clock changes from affecting the logging delay sleep time - // Allocate condition variable - if ((logging_thread_cond_wake_early = OPENSSL_malloc(sizeof(pthread_cond_t))) == NULL) - { - p_scossl_keysinuse_log_error("Failed to allocate condition variable"); - goto cleanup; - } - - if ((pthreadErr = pthread_condattr_init(&attr)) != 0) - { - p_scossl_keysinuse_log_error("Failed to init condition attributes,SYS_%d", pthreadErr); - goto cleanup; - } - - attr_initialized = TRUE; - is_logging = TRUE; - - if ((pthreadErr = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) != 0 || - (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0) - { - p_scossl_keysinuse_log_error("Failed to initialize logging thread condition,SYS_%d", pthreadErr); - is_logging = FALSE; - goto cleanup; - } - - // Block all signals across creation of the logging thread so it inherits a - // fully-blocked mask and never consumes signals intended for other threads - // (for example SIGIO, which the NGINX master relies on for worker channel - // acknowledgements). The previous mask is restored immediately afterwards. - sigfillset(&blockSigSet); - pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); - pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); - pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); - - if (pthreadErr != 0) - { - p_scossl_keysinuse_log_error("Failed to start logging thread,SYS_%d", pthreadErr); - is_logging = FALSE; - goto cleanup; - } - if ((pthreadErr = pthread_atfork(p_scossl_keysinuse_prepare, p_scossl_keysinuse_parent, p_scossl_keysinuse_child)) != 0) @@ -241,15 +263,18 @@ static void p_scossl_keysinuse_init_once() goto cleanup; } - keysinuse_enabled = TRUE; - status = SCOSSL_SUCCESS; + keysinuse_initialized = TRUE; -cleanup: - if (attr_initialized) + if ((keysinuse_process_scope & KEYSINUSE_PROCESS_SCOPE_MAIN) == 0) { - pthread_condattr_destroy(&attr); + status = SCOSSL_SUCCESS; + goto cleanup; } + // Start the logging thread. + status = p_scossl_keysinuse_create_logging_thread(); + +cleanup: if (status != SCOSSL_SUCCESS) { p_scossl_keysinuse_teardown(); @@ -267,7 +292,7 @@ void p_scossl_keysinuse_init() // Acquire all locks to freeze state before fork static void p_scossl_keysinuse_prepare() { - if (!keysinuse_enabled) + if (!keysinuse_initialized) { return; } @@ -304,7 +329,9 @@ static void p_scossl_keysinuse_prepare() closedir(task_dir); // Enable child logging only if no extra threads were found - if (!has_extra_threads && errno == 0) + if (!has_extra_threads && + errno == 0 && + (keysinuse_process_scope & KEYSINUSE_PROCESS_SCOPE_CHILD) != 0) { p_scossl_keysinuse_child_enabled = TRUE; } @@ -329,7 +356,7 @@ static void p_scossl_keysinuse_prepare() // Release all locks in reverse order after fork static void p_scossl_keysinuse_parent() { - if (!keysinuse_enabled) + if (!keysinuse_initialized) { return; } @@ -348,15 +375,10 @@ static void p_scossl_keysinuse_parent() static void p_scossl_keysinuse_child() { SCOSSL_PROV_KEYSINUSE_INFO *pKeysinuseInfo = NULL; - pthread_condattr_t attr; - int pthreadErr; SCOSSL_STATUS status = SCOSSL_FAILURE; int is_parent_logging = is_logging; - BOOL attr_initialized = FALSE; - sigset_t oldSigSet; - sigset_t blockSigSet; - if (!keysinuse_enabled) + if (!keysinuse_initialized) { return; } @@ -411,50 +433,13 @@ static void p_scossl_keysinuse_child() pthread_mutex_unlock(&logging_thread_mutex); - // Only recreate logging thread if it was running in the parent process - if (is_parent_logging && p_scossl_keysinuse_child_enabled) + // Only start the logging thread in the child process if child process + // logging is enabled and either the parent process was logging or + // parent process logging was disabled. + if (p_scossl_keysinuse_child_enabled && + (is_parent_logging || (keysinuse_process_scope & KEYSINUSE_PROCESS_SCOPE_MAIN) == 0)) { - OPENSSL_free(logging_thread_cond_wake_early); - if ((logging_thread_cond_wake_early = OPENSSL_malloc(sizeof(pthread_cond_t))) == NULL) - { - p_scossl_keysinuse_log_error("Failed to allocate condition variable"); - goto cleanup; - } - - if ((pthreadErr = pthread_condattr_init(&attr)) != 0) - { - p_scossl_keysinuse_log_error("Failed to init condition attributes,SYS_%d", pthreadErr); - goto cleanup; - } - - attr_initialized = TRUE; - is_logging = TRUE; - - if ((pthreadErr = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) != 0 || - (pthreadErr = pthread_cond_init(logging_thread_cond_wake_early, &attr)) != 0) - { - p_scossl_keysinuse_log_error("Failed to initialize logging thread condition,SYS_%d", pthreadErr); - is_logging = FALSE; - goto cleanup; - } - - // Block all signals across creation of the logging thread so it - // inherits a fully-blocked mask and never consumes signals intended - // for other threads. The previous mask is restored afterwards. - sigfillset(&blockSigSet); - pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); - pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); - pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); - - if (pthreadErr != 0) - { - p_scossl_keysinuse_log_error("Failed to start logging thread,SYS_%d", pthreadErr); - is_logging = FALSE; - goto cleanup; - } - - keysinuse_enabled = TRUE; - status = SCOSSL_SUCCESS; + status = p_scossl_keysinuse_create_logging_thread(); } cleanup: @@ -465,11 +450,6 @@ static void p_scossl_keysinuse_child() logging_thread_cond_wake_early = NULL; } - if (attr_initialized) - { - pthread_condattr_destroy(&attr); - } - if (status != SCOSSL_SUCCESS) { p_scossl_keysinuse_teardown(); @@ -486,6 +466,7 @@ void p_scossl_keysinuse_teardown() int pthreadErr; keysinuse_enabled = FALSE; + keysinuse_initialized = FALSE; // Finish logging thread if (is_logging) @@ -552,6 +533,15 @@ void p_scossl_keysinuse_set_logging_delay(INT64 delay) } } +void p_scossl_keysinuse_set_process_scope(UINT32 scope) +{ + if (scope != 0 && + (scope & ~(UINT32)KEYSINUSE_PROCESS_SCOPE_BOTH) == 0) + { + keysinuse_process_scope = scope; + } +} + // // KeysInUse info management // diff --git a/SymCryptProvider/src/p_scossl_keysinuse.h b/SymCryptProvider/src/p_scossl_keysinuse.h index f0a29eee..526221a2 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.h +++ b/SymCryptProvider/src/p_scossl_keysinuse.h @@ -38,6 +38,12 @@ void p_scossl_keysinuse_set_logging_id(_In_ const char *id); void p_scossl_keysinuse_set_max_file_size(off_t size); void p_scossl_keysinuse_set_logging_delay(INT64 delay); +#define KEYSINUSE_PROCESS_SCOPE_MAIN 0x1 +#define KEYSINUSE_PROCESS_SCOPE_CHILD 0x2 +#define KEYSINUSE_PROCESS_SCOPE_BOTH (KEYSINUSE_PROCESS_SCOPE_MAIN | KEYSINUSE_PROCESS_SCOPE_CHILD) + +void p_scossl_keysinuse_set_process_scope(UINT32 scope); + // KeysInUse info management SCOSSL_PROV_KEYSINUSE_INFO *p_scossl_keysinuse_info_new(_In_reads_bytes_(cbPublicKey) PBYTE pbPublicKey, SIZE_T cbPublicKey); void p_scossl_keysinuse_info_free(_Inout_ SCOSSL_PROV_KEYSINUSE_INFO *keysinuseInfo); From 27503d8ae98e90ac985d72b7cb87b4533f47dc7e Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 20 Aug 2026 21:56:59 +0000 Subject: [PATCH 5/9] Update documentation and fix typos --- SymCryptProvider/README.md | 3 ++- SymCryptProvider/src/p_scossl_keysinuse.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/SymCryptProvider/README.md b/SymCryptProvider/README.md index a374e26d..f7103e18 100644 --- a/SymCryptProvider/README.md +++ b/SymCryptProvider/README.md @@ -143,4 +143,5 @@ separate section. This section must be referenced in the symcrypt provider secti | - | - | - | | enabled | 0 or 1 to disable or enable keysinuse logging. | 0 | | max_file_size | Maximum size of the file events are written to. May be written as raw byte size or suffixed with KB/MB/GB | 5KB | -| logging_delay_seconds | Duration in seconds between events being written to the file. Any events that happen in between will be aggregate and logged as one event. | error | \ No newline at end of file +| logging_delay_seconds | Duration in seconds between events being written to the file. Any events that happen in between will be aggregate and logged as one event. | error | +| process_scope | Controls which processes run the KeysInUse logging thread when an application forks. Can be
  • main - only the process that first initialized KeysInUse logs
  • child - only forked child processes log
  • both - both the main process and forked children log
May be overridden at runtime by the `KEYSINUSE_PROCESS_SCOPE` environment variable. | both | \ No newline at end of file diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 15a1a9a9..61689e34 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -716,7 +716,7 @@ static void p_scossl_keysinuse_add_use(SCOSSL_PROV_KEYSINUSE_INFO *keysinuseInfo } else { - p_scossl_keysinuse_log_error("Add use failed to accquire mutex,SYS_%d", pthreadErr); + p_scossl_keysinuse_log_error("Add use failed to acquire mutex,SYS_%d", pthreadErr); } } } @@ -1022,7 +1022,7 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) if (pthreadErr != 0) { - p_scossl_keysinuse_log_error("Logging thread failed to accquire mutex,SYS_%d", pthreadErr); + p_scossl_keysinuse_log_error("Logging thread failed to acquire mutex,SYS_%d", pthreadErr); goto cleanup; } @@ -1056,7 +1056,7 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) { if ((pthreadErr = pthread_mutex_lock(&logging_thread_mutex)) != 0) { - p_scossl_keysinuse_log_error("Logging thread failed to accquire mutex,SYS_%d", pthreadErr); + p_scossl_keysinuse_log_error("Logging thread failed to acquire mutex,SYS_%d", pthreadErr); goto cleanup; } From cbf87aad3099bb23e60df94b497e8addee0d59ce Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 20 Aug 2026 23:15:41 +0000 Subject: [PATCH 6/9] Release keysinuse info reference under logging_thread_mutex p_scossl_keysinuse_info_free was called after unlocking logging_thread_mutex. Since the reference release can touch the info's lock (CRYPTO_atomic_add's fallback path), the atfork prepare handler could acquire logging_thread_mutex and fork while that lock was held, letting a child inherit a held lock. Move the reference release inside the mutex so the logging thread never holds an info lock across a fork. The slow log write stays outside the mutex. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b2881f25-ddcb-442e-bb92-49d814cb8300 --- SymCryptProvider/src/p_scossl_keysinuse.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 61689e34..54f764b6 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -1049,9 +1049,10 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) p_scossl_keysinuse_log_error("Failed to lock keysinuse info stack,OPENSSL_%d", ERR_get_error()); } - // Log all pending usage events. logging_thread_mutex is held only around - // during the pKeysinuseInfo update to ensure the logging thread is not - // holding a lock during a fork. + // Log all pending usage events. logging_thread_mutex is held only during + // the pKeysinuseInfo update and reference release to ensure the logging + // thread is not holding a lock during a fork. The slow log write is done + // outside the mutex. while (sk_SCOSSL_PROV_KEYSINUSE_INFO_num(sk_keysinuse_info_pending) > 0) { if ((pthreadErr = pthread_mutex_lock(&logging_thread_mutex)) != 0) @@ -1083,10 +1084,10 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) keysinuseInfoTmp.refCount = -1; } - pthread_mutex_unlock(&logging_thread_mutex); - p_scossl_keysinuse_info_free(pKeysinuseInfo); + pthread_mutex_unlock(&logging_thread_mutex); + if (keysinuseInfoTmp.refCount > 0) { p_scossl_keysinuse_log_notice("%s,%d,%d,%ld,%ld", From 1a41d36f6af9fad14691a68e16f0f89d17accc76 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Thu, 27 Aug 2026 23:46:59 +0000 Subject: [PATCH 7/9] Keep synchronous fault signals deliverable in logging thread The logging thread was created with every signal blocked via sigfillset. Blocking the synchronous fault signals (SIGSEGV, SIGBUS, SIGFPE, SIGILL) is undefined behavior if such a fault occurs in the thread, and can prevent a normal core dump. Unblock the fault signals (and SIGABRT) so a bug in the logging thread still faults and core-dumps normally. These signals are thread-directed, so keeping them deliverable does not affect the isolation of async process-directed signals such as SIGIO. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b2881f25-ddcb-442e-bb92-49d814cb8300 --- SymCryptProvider/src/p_scossl_keysinuse.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 54f764b6..4f516965 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -110,8 +110,8 @@ static void p_scossl_keysinuse_logging_thread_cleanup() } } -// Starts the logging thread and marks keysinuse as enabled. Reainitializes -// lock thread globals and state. On failure caller is repsonsible for cleanup. +// Starts the logging thread and marks keysinuse as enabled. Reinitializes +// lock thread globals and state. On failure caller is responsible for cleanup. static SCOSSL_STATUS p_scossl_keysinuse_create_logging_thread() { pthread_condattr_t attr; @@ -151,8 +151,15 @@ static SCOSSL_STATUS p_scossl_keysinuse_create_logging_thread() } // Block all signals across creation of the logging thread so signal handlers - // never run in the logging thread. + // never run in the logging thread. Keep synchronous fault signals + // deliverable so a bug in the logging thread still faults and core-dumps + // normally (blocking these is undefined behavior if such a fault occurs). sigfillset(&blockSigSet); + sigdelset(&blockSigSet, SIGSEGV); + sigdelset(&blockSigSet, SIGBUS); + sigdelset(&blockSigSet, SIGFPE); + sigdelset(&blockSigSet, SIGILL); + sigdelset(&blockSigSet, SIGABRT); pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL); From 59704e1d8947ad14f7d473534e3594b69ee7b8e1 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Fri, 28 Aug 2026 19:17:54 +0000 Subject: [PATCH 8/9] Move pending stack to file scope to avoid cleanup issues --- SymCryptProvider/src/p_scossl_base.c | 2 +- SymCryptProvider/src/p_scossl_keysinuse.c | 55 ++++++++++++++++++----- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/SymCryptProvider/src/p_scossl_base.c b/SymCryptProvider/src/p_scossl_base.c index 0b67ec97..3d38190b 100644 --- a/SymCryptProvider/src/p_scossl_base.c +++ b/SymCryptProvider/src/p_scossl_base.c @@ -505,7 +505,7 @@ static void p_scossl_start_keysinuse(_In_ const OSSL_CORE_HANDLE *handle) p_scossl_keysinuse_set_logging_delay(atol(confLoggingDelay)); } - // Environment overrides config. Config value is alreday fetched core_get_params above + // Environment overrides config. Config value is already fetched core_get_params above if ((envProcessScope = NCONF_get_string(NULL, NULL, "KEYSINUSE_PROCESS_SCOPE")) != NULL) { confProcessScope = envProcessScope; diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 4f516965..264bb52b 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -55,6 +55,14 @@ DEFINE_STACK_OF(SCOSSL_PROV_KEYSINUSE_INFO); static STACK_OF(SCOSSL_PROV_KEYSINUSE_INFO) *sk_keysinuse_info = NULL; // This lock should be acquired before accessing sk_keysinuse_info static CRYPTO_RWLOCK *sk_keysinuse_info_lock = NULL; +// Stack of keysinuseInfo the logging thread has removed from sk_keysinuse_info +// and is in the process of logging. Held at file scope (rather than on the +// logging thread's stack) so that on fork the child handler can drain any +// in-flight entries. Only the logging thread modifies this in the +// parent; on fork the child handler drains it while single threaded. Allocated +// in init_once and freed when the logging thread exits or on teardown, +// mirroring sk_keysinuse_info. +static STACK_OF(SCOSSL_PROV_KEYSINUSE_INFO) *sk_keysinuse_info_pending = NULL; // To minimize any overhead to crypto operations, all file writes are handled by // logging_thread. This thread periodically pops all pending usage data from @@ -108,6 +116,14 @@ static void p_scossl_keysinuse_logging_thread_cleanup() { p_scossl_keysinuse_log_error("Failed to lock keysinuse info stack,OPENSSL_%d", ERR_get_error()); } + + // Drain and free the pending stack. + while (sk_SCOSSL_PROV_KEYSINUSE_INFO_num(sk_keysinuse_info_pending) > 0) + { + p_scossl_keysinuse_info_free(sk_SCOSSL_PROV_KEYSINUSE_INFO_pop(sk_keysinuse_info_pending)); + } + sk_SCOSSL_PROV_KEYSINUSE_INFO_free(sk_keysinuse_info_pending); + sk_keysinuse_info_pending = NULL; } // Starts the logging thread and marks keysinuse as enabled. Reinitializes @@ -234,7 +250,8 @@ static void p_scossl_keysinuse_init_once() sk_keysinuse_info_lock = CRYPTO_THREAD_lock_new(); sk_keysinuse_info = sk_SCOSSL_PROV_KEYSINUSE_INFO_new_null(); - if (sk_keysinuse_info_lock == NULL || sk_keysinuse_info == NULL) + sk_keysinuse_info_pending = sk_SCOSSL_PROV_KEYSINUSE_INFO_new_null(); + if (sk_keysinuse_info_lock == NULL || sk_keysinuse_info == NULL || sk_keysinuse_info_pending == NULL) { p_scossl_keysinuse_log_error("Failed to create global objects used by keysinuse"); goto cleanup; @@ -424,6 +441,26 @@ static void p_scossl_keysinuse_child() } } + // Drain any in-flight entries the parent's logging thread had moved into + // sk_keysinuse_info_pending but not yet logged. The stack is reused by the + // child's logging thread, so only the entries are released here. + while (sk_SCOSSL_PROV_KEYSINUSE_INFO_num(sk_keysinuse_info_pending) > 0) + { + pKeysinuseInfo = sk_SCOSSL_PROV_KEYSINUSE_INFO_pop(sk_keysinuse_info_pending); + if (pKeysinuseInfo != NULL) + { + pKeysinuseInfo->logPending = FALSE; + pKeysinuseInfo->decryptCounter = 0; + pKeysinuseInfo->signCounter = 0; + pKeysinuseInfo->refCount--; + if (pKeysinuseInfo->refCount == 0) + { + CRYPTO_THREAD_lock_free(pKeysinuseInfo->lock); + OPENSSL_free(pKeysinuseInfo); + } + } + } + if (sk_keysinuse_info_lock != NULL) { CRYPTO_THREAD_unlock(sk_keysinuse_info_lock); @@ -517,8 +554,10 @@ void p_scossl_keysinuse_teardown() CRYPTO_THREAD_lock_free(sk_keysinuse_info_lock); sk_SCOSSL_PROV_KEYSINUSE_INFO_free(sk_keysinuse_info); + sk_SCOSSL_PROV_KEYSINUSE_INFO_free(sk_keysinuse_info_pending); sk_keysinuse_info_lock = NULL; sk_keysinuse_info = NULL; + sk_keysinuse_info_pending = NULL; } // @@ -959,16 +998,13 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) int pthreadErr; int waitStatus; - // Every time the logging loop runs, all pending usage events are popped to sk_keysinuse_info_pending - // to minimize the time sk_keysinuse_info_lock is held. + // Every time the logging loop runs, all pending usage events are popped from + // sk_keysinuse_info to sk_keysinuse_info_pending to minimize + // the time sk_keysinuse_info_lock is held. sk_keysinuse_info_pending is + // allocated in init_once and drained/freed in the logging thread cleanup so + // that the child fork handler can reclaim any in-flight entries. SCOSSL_PROV_KEYSINUSE_INFO *pKeysinuseInfo; SCOSSL_PROV_KEYSINUSE_INFO keysinuseInfoTmp; - STACK_OF(SCOSSL_PROV_KEYSINUSE_INFO) *sk_keysinuse_info_pending = sk_SCOSSL_PROV_KEYSINUSE_INFO_new_null(); - if (sk_keysinuse_info_pending == NULL) - { - p_scossl_keysinuse_log_error("Failed to create pending info stack"); - goto cleanup; - } do { @@ -1111,7 +1147,6 @@ static void *p_scossl_keysinuse_logging_thread_start(ossl_unused void *arg) logging_thread_exit_status = SCOSSL_SUCCESS; cleanup: - sk_SCOSSL_PROV_KEYSINUSE_INFO_free(sk_keysinuse_info_pending); keysinuse_enabled = FALSE; p_scossl_keysinuse_logging_thread_cleanup(); From fac7c63f065e78a377af87ffd559b9ebcbbe0dd5 Mon Sep 17 00:00:00 2001 From: Maxwell Moyer-McKee Date: Fri, 28 Aug 2026 22:39:37 +0000 Subject: [PATCH 9/9] Ignore sigabrt in logging thread --- SymCryptProvider/src/p_scossl_keysinuse.c | 1 - 1 file changed, 1 deletion(-) diff --git a/SymCryptProvider/src/p_scossl_keysinuse.c b/SymCryptProvider/src/p_scossl_keysinuse.c index 264bb52b..aa1354c8 100644 --- a/SymCryptProvider/src/p_scossl_keysinuse.c +++ b/SymCryptProvider/src/p_scossl_keysinuse.c @@ -175,7 +175,6 @@ static SCOSSL_STATUS p_scossl_keysinuse_create_logging_thread() sigdelset(&blockSigSet, SIGBUS); sigdelset(&blockSigSet, SIGFPE); sigdelset(&blockSigSet, SIGILL); - sigdelset(&blockSigSet, SIGABRT); pthread_sigmask(SIG_SETMASK, &blockSigSet, &oldSigSet); pthreadErr = pthread_create(&logging_thread, NULL, p_scossl_keysinuse_logging_thread_start, NULL); pthread_sigmask(SIG_SETMASK, &oldSigSet, NULL);