Files
LithosAnanake/src/rolling_window_of_truth.c
T

1354 lines
52 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
StarForth — Steady-State Virtual Machine Runtime
Copyright (c) 20232025 Robert A. James
All rights reserved.
This file is part of the StarForth project.
Licensed under the StarForth License, Version 1.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
express or implied, including but not limited to the warranties of
merchantability, fitness for a particular purpose, and noninfringement.
See the License for the specific language governing permissions and
limitations under the License.
StarForth — Steady-State Virtual Machine Runtime
Copyright (c) 20232025 Robert A. James
All rights reserved.
This file is part of the StarForth project.
Licensed under the StarForth License, Version 1.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
express or implied, including but not limited to the warranties of
merchantability, fitness for a particular purpose, and noninfringement.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
*** StarForth ***
rolling_window_of_truth.c - Execution history capture and seeding
Implements the rolling window that captures POST execution sequence and
uses it to deterministically seed both hot-words cache and pipelining
context windows.
Key property: Deterministic → Reproducible → Provable
*/
#include "rolling_window_of_truth.h"
#include "../include/vm.h"
#include "../include/rolling_window_knobs.h"
#include "../include/physics_hotwords_cache.h"
#include "../include/physics_pipelining_metrics.h"
#include "../include/log.h"
#include "../include/platform_alloc.h"
#include <string.h>
#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
/**
* @brief Immutable snapshot view of a @c RollingWindowOfTruth.
*
* Provides a read-only window into one of the two double-buffered snapshot
* arrays. Reader threads (the heartbeat, pipelining, and diversity subsystems)
* obtain a @c rolling_window_view_t via @c rolling_window_snapshot_view()
* rather than accessing the live @c RollingWindowOfTruth fields directly;
* this avoids TOCTOU races with the writer thread that calls
* @c rolling_window_record_execution().
*
* Fields mirror the snapshot-time copies of the corresponding
* @c RollingWindowOfTruth members:
* - @c history — pointer into @c snapshot_buffers[idx]; size @c ROLLING_WINDOW_SIZE.
* - @c window_pos — write cursor at snapshot time.
* - @c total_executions — monotonic execution count at snapshot time.
* - @c effective_window_size — adaptive window width at snapshot time.
* - @c is_warm — whether the window had reached ≥ 1024 executions at snapshot.
*/
typedef struct
{
const uint32_t* history;
uint32_t window_pos;
uint64_t total_executions;
uint32_t effective_window_size;
int is_warm;
} rolling_window_view_t;
/**
* @brief Count unique consecutive-pair transitions in a snapshot view.
*
* Scans the @p view's history buffer over the most recent
* @c effective_window_size entries (when warm) or all @c ROLLING_WINDOW_SIZE
* entries (when cold) and counts the number of adjacent pairs
* @c (history[i], history[i+1]) where @c history[i+1] ≠ @c history[i].
*
* This is the primary pattern-diversity metric used by:
* - @c dict_adaptive_optimization_pass() — to choose between heat-aware and
* naive lookup strategies (diversity > 70 → heat-aware).
* - @c rolling_window_run_adaptive_pass() — to decide whether to shrink or
* grow @c effective_window_size.
*
* Returns 0 if @p view is NULL, has no history, or has fewer than 2
* total executions.
*
* @param view Immutable snapshot view; must have @c history != NULL.
* @return Count of unique adjacent-pair transitions in the active window.
*/
static uint64_t rolling_window_measure_diversity_view(const rolling_window_view_t* view)
{
if (!view || !view->history || view->total_executions < 2)
return 0;
uint64_t unique_transitions = 0;
uint32_t prev_word = 0;
int seen_first = 0;
uint32_t scan_limit = view->is_warm ? view->effective_window_size : ROLLING_WINDOW_SIZE;
for (uint32_t i = 0; i < scan_limit; i++)
{
uint32_t idx = (view->window_pos + ROLLING_WINDOW_SIZE - scan_limit + i) % ROLLING_WINDOW_SIZE;
uint32_t current = view->history[idx];
if (seen_first && current != prev_word)
{
unique_transitions++;
}
prev_word = current;
seen_first = 1;
}
return unique_transitions;
}
/**
* @brief Populate a @c rolling_window_view_t from the current double-buffer snapshot.
*
* Reads @c window->snapshot_index with @c __ATOMIC_ACQUIRE to determine which
* of the two snapshot buffers (@c snapshot_buffers[0] or @c [1]) is the most
* recently published. Fills @p view with the corresponding history pointer and
* metadata fields (@c window_pos, @c total_executions, @c effective_window_size,
* @c is_warm). If @p window or @p view is NULL, @p view is zeroed and the
* function returns early.
*
* Callers hold no lock — the acquire barrier on @c snapshot_index ensures they
* see a coherent snapshot that was fully written before
* @c rolling_window_publish_snapshot() released it via @c __ATOMIC_RELEASE.
*
* @param window The @c RollingWindowOfTruth to sample.
* @param view Output @c rolling_window_view_t; zeroed on NULL @p window.
*/
static void rolling_window_snapshot_view(const RollingWindowOfTruth* window,
rolling_window_view_t* view)
{
if (!window || !view)
{
if (view)
memset(view, 0, sizeof(*view));
return;
}
uint32_t idx = __atomic_load_n(&window->snapshot_index, __ATOMIC_ACQUIRE) & 1u;
view->history = window->snapshot_buffers[idx];
view->window_pos = window->snapshot_window_pos[idx];
view->total_executions = window->snapshot_total_executions[idx];
view->effective_window_size = window->snapshot_effective_window_size[idx];
view->is_warm = window->snapshot_is_warm[idx];
}
/**
* @brief Publish the live execution history into the inactive double-buffer slot.
*
* Implements the write side of the lock-free double buffer:
* 1. Identifies the inactive slot as @c write_idx = @c snapshot_index XOR 1.
* 2. Copies @c window->execution_history (the live circular buffer) into
* @c snapshot_buffers[write_idx] via @c memcpy.
* 3. Copies the current @c window_pos, @c total_executions,
* @c effective_window_size, and @c is_warm into the corresponding
* @c snapshot_* parallel arrays.
* 4. Flips @c snapshot_index to @p write_idx with @c __ATOMIC_RELEASE, making
* the new snapshot visible to readers.
* 5. Clears @c snapshot_pending with @c __ATOMIC_RELEASE.
*
* Must only be called from the writer (main VM thread). No-op if @p window,
* @c window->execution_history, or @c snapshot_buffers[0] are NULL.
*
* @param window The @c RollingWindowOfTruth to snapshot.
*/
static void rolling_window_publish_snapshot(RollingWindowOfTruth* window)
{
if (!window || !window->execution_history || !window->snapshot_buffers[0])
return;
uint32_t write_idx = (window->snapshot_index ^ 1u) & 1u;
memcpy(window->snapshot_buffers[write_idx],
window->execution_history,
ROLLING_WINDOW_SIZE * sizeof(uint32_t));
window->snapshot_window_pos[write_idx] = window->window_pos;
window->snapshot_total_executions[write_idx] = window->total_executions;
window->snapshot_effective_window_size[write_idx] = window->effective_window_size;
window->snapshot_is_warm[write_idx] = window->is_warm;
__atomic_store_n(&window->snapshot_index, write_idx, __ATOMIC_RELEASE);
__atomic_store_n(&window->snapshot_pending, 0, __ATOMIC_RELEASE);
}
/**
* @brief Publish a new snapshot only if the @c snapshot_pending flag is set.
*
* Atomically exchanges @c window->snapshot_pending with 0 using
* @c __ATOMIC_ACQ_REL. If the exchanged value was non-zero (meaning
* @c rolling_window_record_execution() has written new data since the last
* publish), calls @c rolling_window_publish_snapshot() to make the current
* state visible to readers. Otherwise returns without copying.
*
* This is the lazy-publish counterpart to @c rolling_window_record_execution()'s
* eager flag set. By batching publish work to read-side call sites (
* @c rolling_window_get_recent_sequence(), @c rolling_window_find_hottest_word(),
* etc.), unnecessary @c memcpy of the full @c ROLLING_WINDOW_SIZE buffer is
* avoided when no new data has arrived.
*
* @param window The @c RollingWindowOfTruth to conditionally snapshot.
* No-op if NULL.
*/
static void rolling_window_publish_snapshot_if_needed(RollingWindowOfTruth* window)
{
if (!window)
return;
uint32_t pending = __atomic_exchange_n(&window->snapshot_pending, 0, __ATOMIC_ACQ_REL);
if (pending)
{
rolling_window_publish_snapshot(window);
}
}
/* ============================================================================
* Rolling Window Implementation
* ============================================================================
*/
/**
* @brief Initialise a @c RollingWindowOfTruth, allocating its three heap buffers.
*
* Allocates and zero-initialises three @c uint32_t arrays, each of
* @c ROLLING_WINDOW_SIZE elements, via @c sf_calloc:
* - @c window->execution_history — the live circular write buffer.
* - @c window->snapshot_buffers[0] — first double-buffer read slot.
* - @c window->snapshot_buffers[1] — second double-buffer read slot.
*
* If any allocation fails, all previously allocated buffers are freed and
* nulled before returning 1. Initialises scalar state:
* - @c window_pos = 0, @c total_executions = 0, @c is_warm = 0 (cold start).
* - @c effective_window_size = @c ROLLING_WINDOW_SIZE (start at maximum).
* - @c snapshot_index = 0, @c snapshot_pending = 1.
* - All @c snapshot_window_pos[] / @c snapshot_total_executions[] / etc. = 0.
* - @c adaptive_check_accumulator = 0, @c adaptive_pending = 0.
*
* Immediately calls @c rolling_window_publish_snapshot() to push the zeroed
* state into both snapshot slots so that any early read-side call before the
* first @c rolling_window_record_execution() gets a coherent (empty) view.
*
* @param window The @c RollingWindowOfTruth to initialise; must not be NULL.
* @return 0 on success; 1 on NULL @p window or allocation failure.
*/
int rolling_window_init(RollingWindowOfTruth* window)
{
if (!window) return -1;
window->execution_history = (uint32_t*)sf_calloc(ROLLING_WINDOW_SIZE, sizeof(uint32_t));
if (!window->execution_history)
{
return -1; /* Malloc failure */
}
window->snapshot_buffers[0] = (uint32_t*)sf_calloc(ROLLING_WINDOW_SIZE, sizeof(uint32_t));
window->snapshot_buffers[1] = (uint32_t*)sf_calloc(ROLLING_WINDOW_SIZE, sizeof(uint32_t));
if (!window->snapshot_buffers[0] || !window->snapshot_buffers[1])
{
sf_free(window->execution_history);
window->execution_history = NULL;
sf_free(window->snapshot_buffers[0]);
sf_free(window->snapshot_buffers[1]);
window->snapshot_buffers[0] = window->snapshot_buffers[1] = NULL;
return -1;
}
window->window_pos = 0;
window->total_executions = 0;
window->is_warm = 0; /* Cold start */
/* Initialize adaptive window sizing: start conservative, shrink if beneficial */
window->effective_window_size = ROLLING_WINDOW_SIZE;
window->last_pattern_diversity = 0;
window->pattern_diversity_check_count = 0;
window->snapshot_index = 0;
window->snapshot_pending = 1;
window->snapshot_window_pos[0] = window->snapshot_window_pos[1] = 0;
window->snapshot_total_executions[0] = window->snapshot_total_executions[1] = 0;
window->snapshot_effective_window_size[0] = window->snapshot_effective_window_size[1] = ROLLING_WINDOW_SIZE;
window->snapshot_is_warm[0] = window->snapshot_is_warm[1] = 0;
window->adaptive_check_accumulator = 0;
window->adaptive_pending = 0;
rolling_window_publish_snapshot(window);
return 0;
}
/**
* @brief Record one word execution in the circular history buffer.
*
* Writes @p word_id into @c window->execution_history[window_pos], advances
* @c window_pos modulo @c ROLLING_WINDOW_SIZE (circular wrap), increments
* @c total_executions, and checks the warm threshold:
* - Once @c total_executions ≥ 1024 and @c is_warm is still 0, sets
* @c is_warm = 1. The 1024-execution threshold is calibrated to the POST
* harness which generates approximately 1450 executions; reaching 1024
* guarantees a statistically meaningful execution distribution before
* diversity or percentile analysis begins.
*
* Sets @c snapshot_pending = 1 to signal that a new snapshot should be
* published on the next read-side access. When the window is warm and
* @c adaptive_check_accumulator reaches @c ADAPTIVE_CHECK_FREQUENCY, also sets
* @c adaptive_pending = 1 to request an @c effective_window_size update.
*
* Called from the interpreter hot path on every word dispatch. All operations
* are O(1) with no locks and no allocations.
*
* @param window The @c RollingWindowOfTruth to record into.
* @param word_id Stable numeric ID of the word just executed.
* @return 0 on success; 1 if @p window or @c execution_history are NULL.
*/
int rolling_window_record_execution(RollingWindowOfTruth* window, uint32_t word_id)
{
if (!window || !window->execution_history)
{
return -1;
}
/* Write word to circular buffer */
window->execution_history[window->window_pos] = word_id;
/* Advance position */
window->window_pos = (window->window_pos + 1) % ROLLING_WINDOW_SIZE;
/* Increment lifetime counter */
window->total_executions++;
/* Mark as warm once we've recorded enough history for pattern recognition */
/* Test harness generates ~1450 executions, so set realistic threshold at 1024 */
if (window->total_executions >= 1024 && !window->is_warm)
{
window->is_warm = 1;
}
/* Signal snapshot consumers that fresh data is available */
window->snapshot_pending = 1;
if (window->is_warm)
{
if (++window->adaptive_check_accumulator >= ADAPTIVE_CHECK_FREQUENCY)
{
window->adaptive_check_accumulator = 0;
window->adaptive_pending = 1;
}
}
return 0;
}
/**
* @brief Copy the most recent @p depth word IDs from the rolling window into @p out_sequence.
*
* Calls @c rolling_window_publish_snapshot_if_needed() then reads the current
* double-buffer snapshot via @c rolling_window_snapshot_view(). Copies at most
* @c min(@p depth, available) entries from the most recent portion of the
* circular buffer into @p out_sequence in chronological order (oldest entry at
* index 0, most recent at index @c actual_depth - 1).
*
* The available count is capped at @c min(@c total_executions, @c ROLLING_WINDOW_SIZE)
* so callers requesting more history than has been recorded get what is available
* rather than stale zeros.
*
* Entries are addressed via:
* @code
* idx = (window_pos - actual_depth + i + ROLLING_WINDOW_SIZE) % ROLLING_WINDOW_SIZE
* @endcode
*
* Returns 0 immediately if @p window, @p out_sequence are NULL or @p depth is 0.
*
* @param window The @c RollingWindowOfTruth to sample.
* @param depth Maximum number of recent word IDs to copy.
* @param out_sequence Caller-allocated output buffer of at least @p depth
* @c uint32_t entries.
* @return Actual number of entries written to @p out_sequence;
* ≤ @p depth.
*/
uint32_t rolling_window_get_recent_sequence(const RollingWindowOfTruth* window,
uint32_t depth,
uint32_t* out_sequence)
{
if (!window || !out_sequence || depth == 0)
{
return 0;
}
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history)
return 0;
/* Cap depth to available data */
uint32_t available = (view.total_executions < ROLLING_WINDOW_SIZE)
? (uint32_t)view.total_executions
: ROLLING_WINDOW_SIZE;
uint32_t actual_depth = (depth > available) ? available : depth;
/* Copy recent sequence in order (most recent last) */
for (uint32_t i = 0; i < actual_depth; i++)
{
/* Index going backwards from current position */
int idx = (int)view.window_pos - (int)actual_depth + (int)i;
if (idx < 0) idx += ROLLING_WINDOW_SIZE;
out_sequence[i] = view.history[idx];
}
return actual_depth;
}
/**
* @brief Return the word ID that appears most frequently in the rolling window.
*
* Obtains the current snapshot, allocates a zero-filled frequency array of
* @p dict_size @c uint32_t counters via @c sf_calloc, and scans all
* @c ROLLING_WINDOW_SIZE history slots to tally occurrences. Then performs a
* linear scan for the maximum and frees the frequency array before returning.
*
* Returns 0 immediately (without allocating) if:
* - @p window is NULL.
* - @c window->is_warm is 0 (not enough history to be meaningful).
* - The snapshot's @c is_warm flag is 0.
* - @c sf_calloc fails.
*
* The returned word ID is the zero-based index into the dictionary's word-ID
* space. Word ID 0 may be returned both as a legitimate hottest word and as
* the not-warm sentinel; callers should check @c rolling_window_is_warm()
* before relying on the result.
*
* @param window The @c RollingWindowOfTruth to analyse.
* @param dict_size Size of the dictionary's word-ID space; bounds the
* frequency array and the scan of the history.
* @return Word ID of the most-frequently-executed word; 0 if
* the window is not warm or allocation fails.
*/
uint32_t rolling_window_find_hottest_word(const RollingWindowOfTruth* window,
uint32_t dict_size)
{
if (!window || !window->is_warm)
{
return 0; /* Not warm yet */
}
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history || !view.is_warm)
return 0;
/* Count frequency of each word */
uint32_t* freq = (uint32_t*)sf_calloc(dict_size, sizeof(uint32_t));
if (!freq) return 0;
for (uint32_t i = 0; i < ROLLING_WINDOW_SIZE; i++)
{
uint32_t word_id = view.history[i];
if (word_id < dict_size)
{
freq[word_id]++;
}
}
/* Find word with highest frequency */
uint32_t hottest_id = 0;
uint32_t max_freq = 0;
for (uint32_t i = 0; i < dict_size; i++)
{
if (freq[i] > max_freq)
{
max_freq = freq[i];
hottest_id = i;
}
}
sf_free(freq);
return hottest_id;
}
/**
* @brief Count how many times the transition (word_a → word_b) occurs in the history.
*
* Scans all @c ROLLING_WINDOW_SIZE consecutive adjacent pairs in the current
* snapshot (wrapping at the buffer end: pair @c i is
* @c (history[i], history[(i+1) % ROLLING_WINDOW_SIZE])) and counts how many
* pairs match @p word_a followed by @p word_b.
*
* Returns 0 if:
* - @p window is NULL.
* - @c window->is_warm is 0 (the history is too short to be meaningful).
* - The snapshot's @c is_warm flag is 0.
*
* Note: because the history is circular and the scan is linear across all
* slots regardless of @c effective_window_size, the count includes transitions
* that span the buffer wrap-around boundary. This is intentional: the full
* @c ROLLING_WINDOW_SIZE is used for transition counting.
*
* @param window The @c RollingWindowOfTruth to query.
* @param word_a ID of the predecessor word.
* @param word_b ID of the successor word.
* @return Count of (word_a → word_b) transitions; 0 if not warm.
*/
uint64_t rolling_window_count_transition(const RollingWindowOfTruth* window,
uint32_t word_a,
uint32_t word_b)
{
if (!window || !window->is_warm)
{
return 0;
}
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history || !view.is_warm)
return 0;
uint64_t count = 0;
/* Scan for (word_a → word_b) transitions */
for (uint32_t i = 0; i < ROLLING_WINDOW_SIZE; i++)
{
uint32_t current = view.history[i];
uint32_t next = view.history[(i + 1) % ROLLING_WINDOW_SIZE];
if (current == word_a && next == word_b)
{
count++;
}
}
return count;
}
/**
* @brief Return whether the rolling window has accumulated enough history to be useful.
*
* Returns @c window->is_warm, which is set to 1 by
* @c rolling_window_record_execution() once @c total_executions ≥ 1024.
* Before that threshold the window is considered "cold" — its contents do not
* yet represent a stable distribution of the program's word usage, and
* analysis functions (@c rolling_window_find_hottest_word(),
* @c rolling_window_count_transition(), adaptive sizing) return early with
* conservative defaults when @c is_warm is 0.
*
* @param window The @c RollingWindowOfTruth to query; NULL → returns 0.
* @return 1 if the window is warm (≥ 1024 executions); 0 otherwise.
*/
int rolling_window_is_warm(const RollingWindowOfTruth* window)
{
if (!window) return 0;
return window->is_warm;
}
/**
* @brief Allocate and return a one-page status summary for a @c RollingWindowOfTruth.
*
* Allocates a 1024-byte buffer via @c sf_malloc and fills it with a
* multi-line report covering:
* - @c total_executions — lifetime word-execution count.
* - @c ROLLING_WINDOW_SIZE — compile-time capacity constant.
* - Warm/cold status: "WARM (representative)" or "COLD (warming up)".
* - @c window_pos / @c ROLLING_WINDOW_SIZE — current write-cursor position.
*
* Values are read from the current double-buffer snapshot obtained via
* @c rolling_window_snapshot_view(); a lazy publish is triggered first by
* @c rolling_window_publish_snapshot_if_needed().
*
* The caller is responsible for calling @c sf_free() on the returned pointer.
* Returns NULL if @p window is NULL or @c sf_malloc fails.
*
* @param window The @c RollingWindowOfTruth to describe.
* @return Heap-allocated NUL-terminated summary string; caller must
* @c sf_free() it. NULL on failure.
*/
char* rolling_window_stats_string(const RollingWindowOfTruth* window)
{
if (!window) return NULL;
char* buf = (char*)sf_malloc(1024);
if (!buf) return NULL;
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
snprintf(buf, 1024,
"Rolling Window of Truth:\n"
" Total Executions: %lu\n"
" Window Size: %u\n"
" Status: %s\n"
" Current Position: %u/%u\n",
(unsigned long)view.total_executions,
ROLLING_WINDOW_SIZE,
view.is_warm ? "WARM (representative)" : "COLD (warming up)",
view.window_pos,
ROLLING_WINDOW_SIZE);
return buf;
}
/**
* @brief Reset the rolling window to an empty cold state without freeing heap memory.
*
* Zeros @c execution_history via @c memset and resets the scalar fields:
* @c window_pos = 0, @c total_executions = 0, @c is_warm = 0. Sets
* @c snapshot_pending = 1 so that the next read will publish an empty snapshot.
*
* The three heap buffers (@c execution_history, @c snapshot_buffers[0/1]) are
* preserved; only their contents are zeroed. Intended for benchmark harnesses
* that need a clean baseline between test runs without the overhead of
* @c rolling_window_cleanup() + @c rolling_window_init().
*
* @param window The @c RollingWindowOfTruth to reset; no-op if NULL or
* @c execution_history is NULL.
*/
void rolling_window_reset(RollingWindowOfTruth* window)
{
if (!window || !window->execution_history)
{
return;
}
memset(window->execution_history, 0, ROLLING_WINDOW_SIZE * sizeof(uint32_t));
window->window_pos = 0;
window->total_executions = 0;
window->is_warm = 0;
window->snapshot_pending = 1;
}
/**
* @brief Free all heap memory held by a @c RollingWindowOfTruth.
*
* Calls @c sf_free() on @c execution_history and both @c snapshot_buffers[],
* setting each to NULL after freeing. Does not zero the scalar fields; the
* caller must not use @p window after this call without a subsequent
* @c rolling_window_init().
*
* No-op if @p window is NULL. Each buffer pointer is individually NULL-checked
* before freeing (defensive against partial-init failures from
* @c rolling_window_init()).
*
* @param window The @c RollingWindowOfTruth to clean up; no-op if NULL.
*/
void rolling_window_cleanup(RollingWindowOfTruth* window)
{
if (!window) return;
if (window->execution_history)
{
sf_free(window->execution_history);
window->execution_history = NULL;
}
if (window->snapshot_buffers[0])
{
sf_free(window->snapshot_buffers[0]);
window->snapshot_buffers[0] = NULL;
}
if (window->snapshot_buffers[1])
{
sf_free(window->snapshot_buffers[1]);
window->snapshot_buffers[1] = NULL;
}
}
/* ============================================================================
* Seeding Functions: Apply rolling window data to optimization systems
* ============================================================================
*/
/**
* @brief Warm-start the hot-words cache from POST execution history.
*
* Called exactly once after POST completes and before the REPL begins, so
* that the first interactive command benefits from cache entries derived from
* real POST execution data rather than starting cold. The algorithm:
*
* 1. Allocates a zero-filled frequency array of @c DICTIONARY_SIZE @c uint32_t
* counters via @c sf_calloc.
* 2. Obtains the current snapshot and tallies occurrences of each word ID
* over all @c ROLLING_WINDOW_SIZE history slots.
* 3. Computes a promotion threshold: words appearing more than the average
* frequency across all non-zero-frequency words are considered "hot".
* 4. Iterates over the frequency array; for each word above the threshold,
* looks up the live @c DictEntry via @c vm_dictionary_lookup_by_word_id()
* (under @c vm->dict_lock), updates @c entry->execution_heat to the
* observed frequency count, and calls @c hotwords_cache_promote() to insert
* it into the cache.
* 5. Logs the count of promoted words at @c LOG_INFO.
*
* No-op if @p window is NULL, not warm, @p vm is NULL, or @c vm->hotwords_cache
* is NULL. Also no-op if the snapshot's @c is_warm flag is 0.
*
* @param window Warm @c RollingWindowOfTruth containing POST execution data.
* @param vm The VM whose @c hotwords_cache is seeded and whose
* @c dict_lock is held during @c DictEntry updates.
*/
void rolling_window_seed_hotwords_cache(const RollingWindowOfTruth* window,
struct VM* vm)
{
if (!window || !vm || !window->is_warm || !vm->hotwords_cache)
{
return; /* Window not ready or no cache */
}
/* Count frequency of each word in rolling window */
uint32_t *freq = (uint32_t *)sf_calloc(DICTIONARY_SIZE, sizeof(uint32_t));
if (!freq) return;
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history || !view.is_warm)
{
sf_free(freq);
return;
}
for (uint32_t i = 0; i < ROLLING_WINDOW_SIZE; i++)
{
uint32_t word_id = view.history[i];
if (word_id < DICTIONARY_SIZE)
{
freq[word_id]++;
}
}
/* Promote hot words to cache: threshold = median frequency */
uint32_t total_hot_words = 0;
uint32_t sum_freq = 0;
uint32_t count_nonzero = 0;
for (uint32_t i = 0; i < DICTIONARY_SIZE; i++)
{
if (freq[i] > 0)
{
sum_freq += freq[i];
count_nonzero++;
}
}
/* Threshold: promote words above average frequency */
uint32_t threshold = (count_nonzero > 0) ? (sum_freq / count_nonzero) : 1;
sf_mutex_lock(&vm->dict_lock);
for (uint32_t i = 0; i < DICTIONARY_SIZE; i++)
{
if (freq[i] > threshold)
{
/* Lookup dictionary entry by stable word_id mapping */
DictEntry *entry = vm_dictionary_lookup_by_word_id(vm, i);
if (entry)
{
/* Update execution heat and promote to cache */
entry->execution_heat = (cell_t)freq[i];
hotwords_cache_promote(vm->hotwords_cache, entry);
total_hot_words++;
}
}
}
sf_mutex_unlock(&vm->dict_lock);
sf_free(freq);
log_message(LOG_INFO,
"Seeded hot-words cache with %u words from rolling window POST execution",
total_hot_words);
}
/**
* @brief Warm-start pipelining context windows from POST execution history.
*
* Called exactly once after POST completes and before the REPL begins,
* to populate each word's @c transition_metrics context window with real
* predecessor data so that speculative prefetch can begin making informed
* decisions on the first interactive command. The algorithm:
*
* 1. Allocates a sliding @p context[] buffer of @c context_window_size
* @c uint32_t entries (max(@c TRANSITION_WINDOW_SIZE, 2)) and initialises
* it with the first @c context_window_size entries from the history.
* 2. Scans all @c ROLLING_WINDOW_SIZE consecutive adjacent pairs in the
* snapshot:
* - Looks up @c current_entry = @c vm_dictionary_lookup_by_word_id(current_id)
* (under @c vm->dict_lock, released immediately).
* - If @c current_entry->transition_metrics is set, calls
* @c transition_metrics_record_context() with the current @p context[]
* and @c next_id to populate the Phase 1 context-transition counter.
* - Shifts the @p context[] buffer left by one, appending @c next_id at
* the right end (sliding window advance).
* 3. Logs seeded-words and seeded-transitions counts at @c LOG_INFO.
*
* No-op if @p window is NULL, not warm, @p vm is NULL, @c ENABLE_PIPELINING
* is 0, or the context buffer allocation fails.
*
* @param window Warm @c RollingWindowOfTruth containing POST execution data.
* @param vm The VM whose per-word @c transition_metrics structures are
* seeded; @c dict_lock is briefly held per lookup.
*/
void rolling_window_seed_pipelining_context(const RollingWindowOfTruth* window,
struct VM* vm)
{
if (!window || !vm || !window->is_warm || !ENABLE_PIPELINING)
{
return; /* Window not ready or pipelining disabled */
}
uint32_t context_window_size = (TRANSITION_WINDOW_SIZE < 2) ? 2 : TRANSITION_WINDOW_SIZE;
uint32_t seeded_words = 0;
uint32_t seeded_transitions = 0;
/* Allocate context window for replaying */
uint32_t *context = (uint32_t *)sf_calloc(context_window_size, sizeof(uint32_t));
if (!context)
return;
uint32_t init_count = (context_window_size < ROLLING_WINDOW_SIZE) ? context_window_size : ROLLING_WINDOW_SIZE;
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history || !view.is_warm)
{
sf_free(context);
return;
}
for (uint32_t i = 0; i < init_count && i < ROLLING_WINDOW_SIZE; i++)
{
context[i] = view.history[i];
}
/* Scan rolling window and seed context windows by replaying transitions */
for (uint32_t i = 0; i < ROLLING_WINDOW_SIZE; i++)
{
uint32_t current_id = view.history[i];
uint32_t next_id = view.history[(i + 1) % ROLLING_WINDOW_SIZE];
if (current_id < DICTIONARY_SIZE && next_id < DICTIONARY_SIZE)
{
/* Lookup dictionary entry for current word by stable ID */
DictEntry *current_entry = NULL;
sf_mutex_lock(&vm->dict_lock);
current_entry = vm_dictionary_lookup_by_word_id(vm, current_id);
sf_mutex_unlock(&vm->dict_lock);
if (current_entry && current_entry->transition_metrics)
{
/* Record this transition with current context */
transition_metrics_record_context(
current_entry->transition_metrics,
context,
context_window_size,
next_id,
DICTIONARY_SIZE);
seeded_transitions++;
/* Update context window by shifting and adding current word */
for (uint32_t j = 0; j < context_window_size - 1; j++)
{
context[j] = context[j + 1];
}
context[context_window_size - 1] = next_id;
seeded_words++;
}
}
}
sf_free(context);
log_message(LOG_INFO,
"Seeded pipelining context windows: %u words, %u transitions from POST execution",
seeded_words,
seeded_transitions);
}
/* ============================================================================
* Bootstrap Data Analysis Implementation
* ============================================================================
*/
/**
* @brief Export up to @p max_count entries from the execution history in chronological order.
*
* Linearises the internal circular buffer so callers get entries from oldest
* to newest without knowing the internal @c window_pos cursor. Two cases:
*
* - **Pre-wrap** (@c total_executions < @c ROLLING_WINDOW_SIZE): the buffer
* has not yet wrapped; entries 0..@c total_executions-1 are valid and are
* copied contiguously from @c history[0].
*
* - **Post-wrap** (@c total_executions ≥ @c ROLLING_WINDOW_SIZE): the oldest
* entry is at @c history[window_pos]. Two @c memcpy calls stitch the
* two halves together:
* 1. @c history[window_pos .. ROLLING_WINDOW_SIZE-1] (first part)
* 2. @c history[0 .. window_pos-1] (second part, if @p export_count >
* @c first_part)
*
* @p export_count is @c min(@c total_executions, @p max_count). Returns 0 if
* @p window, @p out_sequence are NULL, @p max_count is 0, the snapshot has no
* history, or @c export_count is 0.
*
* @param window The @c RollingWindowOfTruth to export.
* @param out_sequence Caller-allocated output buffer of at least @p max_count
* @c uint32_t entries.
* @param max_count Maximum number of entries to export.
* @return Actual number of entries written to @p out_sequence.
*/
uint64_t rolling_window_export_execution_history(const RollingWindowOfTruth* window,
uint32_t* out_sequence,
uint64_t max_count)
{
if (!window || !out_sequence || max_count == 0)
{
return 0;
}
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history)
return 0;
/* Determine how many entries to export (capped by caller buffer) */
uint64_t export_count = (view.total_executions < max_count)
? view.total_executions
: max_count;
if (export_count == 0)
return 0;
/* If we haven't wrapped around yet (total < ROLLING_WINDOW_SIZE),
* just copy from start to current position */
if (view.total_executions < ROLLING_WINDOW_SIZE)
{
memcpy(out_sequence,
view.history,
export_count * sizeof(uint32_t));
return export_count;
}
/* Otherwise, linearize the circular buffer:
* Copy from window_pos to end, then from start to window_pos */
uint64_t first_part = ROLLING_WINDOW_SIZE - view.window_pos;
memcpy(out_sequence,
&view.history[view.window_pos],
first_part * sizeof(uint32_t));
if (export_count > first_part)
{
memcpy(&out_sequence[first_part],
view.history,
(export_count - first_part) * sizeof(uint32_t));
}
return export_count;
}
/**
* @brief Estimate the fraction of transitions that carry unique contextual information.
*
* Measures how many of the recorded adjacent-pair transitions produce a non-zero
* hash when the preceding @p test_window_size word IDs are incorporated:
* @code
* pattern_hash = (... ((word[i-ws] × 31 + word[i-ws+1]) × 31 + ...) × 31 + word[i+1])
* mod 0x7FFFFFFF
* @endcode
* A non-zero hash is counted as a "captured" pattern. The capture rate is:
* @code
* rate = patterns_seen / total_transitions × 100 (clamped to [0, 100])
* @endcode
*
* In practice, hash collisions with the zero sentinel are extraordinarily rare
* (probability ~1/2^31), so nearly all transitions contribute to @c patterns_seen.
* The function thus measures whether @p test_window_size is wide enough to produce
* a non-trivial hash — a proxy for whether the window introduces meaningful context
* beyond a single predecessor.
*
* Linearises the circular buffer via @c rolling_window_export_execution_history()
* before scanning. Returns 0.0 if @p window or @p test_window_size < 1, if the
* linearised history has fewer than 2 entries, or if @c sf_malloc fails.
*
* @param window The @c RollingWindowOfTruth to analyse.
* @param test_window_size Number of preceding word IDs to fold into each pattern hash.
* @return Capture rate as a percentage in [0.0, 100.0].
*/
double rolling_window_pattern_capture_rate(const RollingWindowOfTruth* window,
uint32_t test_window_size)
{
if (!window || test_window_size < 1)
{
return 0.0;
}
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
if (!view.history || view.total_executions < 2)
return 0.0;
/* Allocate space for linearized history */
uint64_t history_size = (view.total_executions < (uint64_t)ROLLING_WINDOW_SIZE)
? view.total_executions
: (uint64_t)ROLLING_WINDOW_SIZE;
uint32_t* linear_history = (uint32_t *)sf_malloc(history_size * sizeof(uint32_t));
if (!linear_history)
return 0.0;
uint64_t actual_history = rolling_window_export_execution_history(
window, linear_history, history_size);
if (actual_history < 2)
{
sf_free(linear_history);
return 0.0;
}
/* Hash set to track unique (context, next_word) patterns
* Using simple counting: patterns seen / potential patterns */
uint64_t patterns_seen = 0;
uint64_t total_transitions = actual_history - 1;
/* Simple pattern tracking: just count unique (word_a, word_b, ..., word_n) → word_next */
/* For efficiency, we'll use a simplified approach: count how many transitions
* can be uniquely identified with the given context window size */
for (uint64_t i = 0; i < actual_history - 1; i++)
{
/* Create a simple hash of the context window and next word
* This is a rough approximation of pattern uniqueness */
uint32_t pattern_hash = 0;
/* Incorporate the sliding context window into hash */
for (uint32_t j = 0; j < test_window_size && i >= j; j++)
{
pattern_hash = (pattern_hash * 31 + linear_history[i - j]) % 0x7FFFFFFF;
}
pattern_hash = (pattern_hash * 31 + linear_history[i + 1]) % 0x7FFFFFFF;
/* Count this as a valid pattern observation */
if (pattern_hash > 0)
patterns_seen++;
}
sf_free(linear_history);
/* Return percentage: how many transitions had sufficient context to be meaningful */
if (total_transitions == 0)
return 0.0;
double capture_rate = (double)patterns_seen / (double)total_transitions * 100.0;
return (capture_rate > 100.0) ? 100.0 : capture_rate;
}
/* ============================================================================
* Adaptive Window Sizing (Continuous Self-Tuning)
* ============================================================================
*
* During execution, the rolling window continuously measures whether it's
* capturing diminishing returns. If pattern diversity plateaus, the effective
* window size shrinks automatically.
*/
/**
* @brief Return the current pattern diversity score for the rolling window.
*
* Triggers a lazy snapshot publish via @c rolling_window_publish_snapshot_if_needed(),
* acquires a read-side view via @c rolling_window_snapshot_view(), and delegates
* to the internal @c rolling_window_measure_diversity_view() to count the number
* of unique adjacent-pair transitions in the active window.
*
* Higher diversity means the window is observing a wide variety of word sequences,
* indicating that the execution pattern is volatile and heat-stratified lookup is
* more beneficial than stable-pattern nearest-first lookup. Lower diversity means
* the program is executing a narrow, repetitive sequence — pattern knowledge is
* concentrated and reliable.
*
* Used by @c dict_adaptive_optimization_pass() to select lookup strategy
* (threshold = 70 unique transitions) and by @c rolling_window_run_adaptive_pass()
* to calibrate @c effective_window_size.
*
* @param window The @c RollingWindowOfTruth to measure.
* @return Count of unique adjacent-pair transitions in the active window;
* 0 if the window has fewer than 2 executions or @p window is NULL.
*/
uint64_t rolling_window_measure_diversity(const RollingWindowOfTruth* window)
{
rolling_window_publish_snapshot_if_needed((RollingWindowOfTruth*)window);
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
return rolling_window_measure_diversity_view(&view);
}
/**
* @brief Adjust @c effective_window_size based on pattern diversity growth rate.
*
* Runs the core adaptive window logic using the snapshot @p view. Measures the
* current diversity via @c rolling_window_measure_diversity_view(), computes the
* fractional growth rate relative to @c window->last_pattern_diversity in Q16.16
* fixed-point:
* @code
* growth_rate_q48 = (diversity_delta << 16) / last_pattern_diversity
* @endcode
* and compares it against a threshold:
* @code
* threshold_q48 = (ADAPTIVE_GROWTH_THRESHOLD << 16) / 100
* @endcode
*
* **Shrink path** (@c growth_rate_q48 < @c threshold_q48): diversity is
* plateauing; reducing the window saves scan work. The new size is:
* @code
* new_size = max(effective_window_size × ADAPTIVE_SHRINK_RATE / 100,
* ADAPTIVE_MIN_WINDOW_SIZE)
* @endcode
* Already at @c ADAPTIVE_MIN_WINDOW_SIZE → no change (logs "Floor Reached").
*
* **Grow path** (@c growth_rate_q48 ≥ @c threshold_q48): diversity is still
* increasing; the larger window is capturing new patterns. The new size is:
* @code
* new_size = min(effective_window_size × 100 / ADAPTIVE_SHRINK_RATE,
* ROLLING_WINDOW_SIZE)
* @endcode
* Already at @c ROLLING_WINDOW_SIZE → no change (logs "Ceiling Reached").
*
* On the very first call after init, @c last_pattern_diversity == 0; only the
* baseline is recorded and the function returns without resizing. All decisions
* are logged at @c LOG_DEBUG for offline analysis.
*
* No-op if @p window or @p view are NULL or the view is not warm.
*
* @param window The @c RollingWindowOfTruth whose @c effective_window_size
* and @c last_pattern_diversity are updated.
* @param view Current immutable snapshot; must be warm.
*/
static void rolling_window_run_adaptive_pass(RollingWindowOfTruth* window,
const rolling_window_view_t* view)
{
if (!window || !view || !view->history || !view->is_warm)
return;
uint64_t current_diversity = rolling_window_measure_diversity_view(view);
window->pattern_diversity_check_count++;
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Check #%lu] total_execs=%lu window_size=%u diversity=%lu",
window->pattern_diversity_check_count,
(unsigned long)view->total_executions,
window->effective_window_size,
current_diversity);
if (window->last_pattern_diversity == 0)
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Baseline] Recording initial diversity=%lu as baseline",
current_diversity);
window->last_pattern_diversity = current_diversity;
return;
}
uint64_t diversity_delta = (current_diversity > window->last_pattern_diversity)
? (current_diversity - window->last_pattern_diversity)
: 0;
uint64_t growth_rate_q48 = (window->last_pattern_diversity > 0)
? ((diversity_delta << 16) / window->last_pattern_diversity)
: 0;
uint64_t threshold_q48 = ((uint64_t)ADAPTIVE_GROWTH_THRESHOLD << 16) / 100;
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Calc] delta=%lu last=%lu growth_rate_q48=%lx threshold_q48=%lx",
diversity_delta,
window->last_pattern_diversity,
(unsigned long)growth_rate_q48,
(unsigned long)threshold_q48);
if (growth_rate_q48 < threshold_q48)
{
if (window->effective_window_size > ADAPTIVE_MIN_WINDOW_SIZE)
{
uint32_t new_size = (window->effective_window_size * ADAPTIVE_SHRINK_RATE) / 100;
if (new_size < ADAPTIVE_MIN_WINDOW_SIZE)
new_size = ADAPTIVE_MIN_WINDOW_SIZE;
if (new_size < window->effective_window_size)
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[SHRINK] %u → %u (growth_rate=%lu < %lu threshold, diversity_delta=%lu)",
window->effective_window_size,
new_size,
(unsigned long)growth_rate_q48,
(unsigned long)threshold_q48,
diversity_delta);
window->effective_window_size = new_size;
}
else
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Calc Error] new_size=%u >= current=%u, not changing",
new_size,
window->effective_window_size);
}
}
else
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Floor Reached] already at minimum size %u, cannot shrink further",
window->effective_window_size);
}
}
else if (growth_rate_q48 >= threshold_q48)
{
if (window->effective_window_size < ROLLING_WINDOW_SIZE)
{
uint32_t growth_factor = (100 * 100) / ADAPTIVE_SHRINK_RATE;
uint32_t new_size = (window->effective_window_size * growth_factor) / 100;
if (new_size > ROLLING_WINDOW_SIZE)
new_size = ROLLING_WINDOW_SIZE;
if (new_size > window->effective_window_size)
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[GROW] %u → %u (growth_rate=%lu >= %lu threshold, diversity_delta=%lu)",
window->effective_window_size,
new_size,
(unsigned long)growth_rate_q48,
(unsigned long)threshold_q48,
diversity_delta);
window->effective_window_size = new_size;
}
else
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Calc Error] new_size=%u <= current=%u, not changing",
new_size,
window->effective_window_size);
}
}
else
{
log_message(LOG_DEBUG,
"ADAPTIVE_WINDOW[Ceiling Reached] already at maximum size %u, cannot grow further",
window->effective_window_size);
}
}
window->last_pattern_diversity = current_diversity;
}
/**
* @brief Drain both pending flags — snapshot and adaptive — in one call.
*
* Called from both @c rolling_window_check_adaptive_shrink() and
* @c rolling_window_service() as the single internal dispatch point:
* 1. Calls @c rolling_window_publish_snapshot_if_needed() to flush any pending
* execution history into the double-buffer snapshot.
* 2. Checks @c window->adaptive_pending; if set, acquires a view, runs
* @c rolling_window_run_adaptive_pass(), then clears @c adaptive_pending.
*
* No-op if @p window is NULL.
*
* @param window The @c RollingWindowOfTruth to service.
*/
static void rolling_window_service_internal(RollingWindowOfTruth* window)
{
if (!window)
return;
rolling_window_publish_snapshot_if_needed(window);
if (!window->adaptive_pending)
return;
rolling_window_view_t view;
rolling_window_snapshot_view(window, &view);
rolling_window_run_adaptive_pass(window, &view);
window->adaptive_pending = 0;
}
/**
* @brief Force an immediate adaptive window check, bypassing the accumulator gate.
*
* Sets @c window->adaptive_pending = 1 unconditionally and then calls
* @c rolling_window_service_internal() to drain it immediately. This allows
* external callers (e.g. the heartbeat timer or a FORTH word) to trigger a
* window-size evaluation outside the normal @c ADAPTIVE_CHECK_FREQUENCY cadence
* — for example, after a major workload transition or at benchmark checkpoints.
*
* If the window is not warm the adaptive pass is a no-op internally.
*
* @param window The @c RollingWindowOfTruth to check; no-op if NULL.
*/
void rolling_window_check_adaptive_shrink(RollingWindowOfTruth* window)
{
if (!window)
return;
window->adaptive_pending = 1;
rolling_window_service_internal(window);
}
/**
* @brief Service the rolling window — publish pending snapshots and run adaptive sizing.
*
* The public entry point for the rolling window's periodic maintenance. Intended
* to be called from the heartbeat tick handler or the REPL idle loop at a rate
* determined by the caller (typically 100 Hz from the APIC timer ISR via
* @c heartbeat_tick()). Delegates entirely to @c rolling_window_service_internal():
* - Flushes the pending snapshot if @c snapshot_pending is set.
* - Runs @c rolling_window_run_adaptive_pass() if @c adaptive_pending is set.
*
* Unlike @c rolling_window_check_adaptive_shrink(), this function does NOT force
* @c adaptive_pending = 1; it only services flags that the record path has already
* set. The normal adaptive-pass cadence is therefore governed entirely by
* @c ADAPTIVE_CHECK_FREQUENCY executions between checks.
*
* @param window The @c RollingWindowOfTruth to service; no-op if NULL.
*/
void rolling_window_service(RollingWindowOfTruth* window)
{
rolling_window_service_internal(window);
}