#!/bin/bash
#
# StarForth — Steady-State Virtual Machine Runtime
#
# Copyright (c) 2023–2025 Robert A. James
# All rights reserved.
#
# This file is part of the StarForth project.
#
# Licensed under the StarForth License, Version 1.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
# https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
#
# This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
# express or implied, including but not limited to the warranties of
# merchantability, fitness for a particular purpose, and noninfringement.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# StarForth — Steady-State Virtual Machine Runtime
# Copyright (c) 2023–2025 Robert A. James
# All rights reserved.
#
# This file is part of the StarForth project.
#
# Licensed under the StarForth License, Version 1.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
# https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
#
# This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
# express or implied, including but not limited to the warranties of
# merchantability, fitness for a particular purpose, and noninfringement.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
################################################################################
#
# StarForth Factorial DoE with Heartbeat Observability
#
# FOCUSED EXPERIMENT: Top 5 Configurations (Stage 1 Winners)
#
# Design:
# ───────────────────────────────────────────────────────────────────────────
#
# This script executes a focused factorial design of experiments:
# - 5 elite configurations (identified from Stage 1 baseline DoE)
# - Each tested N times (default: 150 runs per config)
# - Total runs: 5 × 150 = 750 runs (statistically significant for correlation analysis)
# - All heartbeat dynamics captured: tick interval, jitter, convergence
#
# Elite Configurations (from 3200-run Stage 1 analysis):
# ───────────────────────────────────────────────────────────────────────────
# 1. 1_0_1_1_1_0 - Strong performer, minimal inference overhead
# 2. 1_0_1_1_1_1 - Similar to #1, with decay inference
# 3. 1_1_0_1_1_1 - Alternative: heat + window + pipelining + all inference
# 4. 1_0_1_0_1_0 - Lean config: heat + decay only
# 5. 0_1_1_0_1_1 - Contrast: window + decay + inference without heat tracking
#
# Factors (binary: 0 or 1):
# ───────────────────────────────────────────────────────────────────────────
# L1 = ENABLE_LOOP_1_HEAT_TRACKING (execution heat counting)
# L2 = ENABLE_LOOP_2_ROLLING_WINDOW (rolling window history)
# L3 = ENABLE_LOOP_3_LINEAR_DECAY (exponential heat decay)
# L4 = ENABLE_LOOP_4_PIPELINING_METRICS (word transition tracking)
# L5 = ENABLE_LOOP_5_WINDOW_INFERENCE (window width inference)
# L6 = ENABLE_LOOP_6_DECAY_INFERENCE (decay slope inference)
#
# Heartbeat Metrics Collected:
# ───────────────────────────────────────────────────────────────────────────
# - Per-tick interval (ns)
# - Tick jitter (CV, outliers)
# - Cache hit % stability (rolling window)
# - Window convergence trajectory
# - Decay slope fitting progress
# - Load→heartbeat response coupling
# - Overall stability score
#
# Output Structure:
# ───────────────────────────────────────────────────────────────────────────
# //
# ├── experiment_results_heartbeat.csv (250 rows, 60+ columns)
# ├── experiment_summary.txt (metadata, timing, notes)
# ├── test_matrix.txt (randomized run order)
# ├── configuration_manifest.txt (top 5 configs, rationale)
# ├── run_logs/ (per-run execution logs)
# │ ├── 1_0_1_1_1_0_run_1.log
# │ ├── 1_0_1_1_1_1_run_1.log
# │ └── ... (250 total logs)
# └── stability_analysis.txt (preliminary analysis results_run_01_2025_12_08)
#
# Expected Runtime:
# ───────────────────────────────────────────────────────────────────────────
# - 50 runs/config (250 total): ~45 min - 1.5 hours
# - 100 runs/config (500 total): ~1.5 - 3 hours
#
# Usage:
# ./scripts/run_factorial_doe_with_heartbeat.sh EXPERIMENT_LABEL
# ./scripts/run_factorial_doe_with_heartbeat.sh --runs-per-config 100 HEARTBEAT_HW
#
# Example:
# ./scripts/run_factorial_doe_with_heartbeat.sh 2025_11_20_HEARTBEAT_TOP5
# → stores in /home/rajames/CLionProjects/StarForth-DoE/experiments/2025_11_20_HEARTBEAT_TOP5
#
################################################################################
set -e
################################################################################
# Parse Arguments
################################################################################
RUNS_PER_CONFIG=150
OUTPUT_DIR=""
EXPERIMENTS_BASE="/home/rajames/CLionProjects/StarForth-DoE/experiments"
while [[ $# -gt 0 ]]; do
case $1 in
--runs-per-config)
RUNS_PER_CONFIG="$2"
shift 2
;;
-*)
echo "Unknown option: $1"
echo "Usage: $0 [--runs-per-config N] EXPERIMENT_LABEL"
exit 1
;;
*)
OUTPUT_DIR="$1"
shift
;;
esac
done
if [ -z "${OUTPUT_DIR}" ]; then
echo "Usage: $0 [--runs-per-config N] EXPERIMENT_LABEL"
echo ""
echo "Examples:"
echo " $0 2025_11_20_HEARTBEAT_TOP5 # 150 runs per config (750 total)"
echo " $0 --runs-per-config 200 HEARTBEAT_FULL # 200 runs per config (1000 total)"
exit 1
fi
if ! [[ "${RUNS_PER_CONFIG}" =~ ^[0-9]+$ ]] || [ "${RUNS_PER_CONFIG}" -lt 1 ]; then
echo "Error: --runs-per-config must be a positive integer"
exit 1
fi
# Paths
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
BUILD_DIR="${REPO_ROOT}/build"
BUILD_PROFILE="fastest"
# Ensure base directory exists
mkdir -p "${EXPERIMENTS_BASE}" || {
echo "Error: unable to create experiments base at ${EXPERIMENTS_BASE}"
exit 1
}
OUTPUT_DIR="${EXPERIMENTS_BASE%/}/${OUTPUT_DIR}"
LOG_DIR="${OUTPUT_DIR}/run_logs"
RESULTS_CSV="${OUTPUT_DIR}/experiment_results_heartbeat.csv"
SUMMARY_LOG="${OUTPUT_DIR}/experiment_summary.txt"
TEST_MATRIX="${OUTPUT_DIR}/test_matrix.txt"
CONFIG_MANIFEST="${OUTPUT_DIR}/configuration_manifest.txt"
ANALYSIS_LOG="${OUTPUT_DIR}/stability_analysis.txt"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
mkdir -p "${LOG_DIR}" "${OUTPUT_DIR}"
################################################################################
# Logging Functions
################################################################################
log_header() {
echo -e "${BLUE}═══════════════════════════════════════════════════════════${NC}" >&2
echo -e "${BLUE}$1${NC}" >&2
echo -e "${BLUE}═══════════════════════════════════════════════════════════${NC}" >&2
}
log_success() {
echo -e "${GREEN}✓ $1${NC}" >&2
}
log_error() {
echo -e "${RED}✗ $1${NC}" >&2
}
log_info() {
echo -e "${YELLOW}→ $1${NC}" >&2
}
log_section() {
echo -e "\n${BLUE}>>> $1${NC}" >&2
}
timestamp() {
date +"%Y-%m-%dT%H:%M:%S"
}
runtime_seconds() {
local start=$1
local end=$2
echo $((end - start))
}
format_duration() {
local seconds=$1
local hours=$((seconds / 3600))
local minutes=$(((seconds % 3600) / 60))
local secs=$((seconds % 60))
printf "%dh %dm %ds" $hours $minutes $secs
}
extract_final_csv_row() {
local log_file=$1
if [ ! -f "${log_file}" ]; then
return 1
fi
awk '!/^HB,/ && NF {line=$0} END {if (length(line)) print line}' "${log_file}"
}
extract_heartbeat_timeseries() {
local log_file=$1
local out_file=$2
if [ ! -f "${log_file}" ]; then
return 1
fi
grep '^HB,' "${log_file}" > "${out_file}" || true
}
extract_heartbeat_detail_csv() {
local log_file=$1
local out_file=$2
if [ ! -f "${log_file}" ]; then
return 1
fi
awk 'BEGIN{emit=0} /^run_id,config/ {emit=1} emit && NF' "${log_file}" > "${out_file}" || true
}
################################################################################
# Top 5 Configurations (Elite Subset from Stage 1)
################################################################################
declare -a TOP5_CONFIGS=(
"1_0_1_1_1_0" # Config 1: F1 F3 F4 F5 (strong, minimal inference)
"1_0_1_1_1_1" # Config 2: F1 F3 F4 F5 F6 (add decay inference)
"1_1_0_1_1_1" # Config 3: F1 F2 F4 F5 F6 (alternative path)
"1_0_1_0_1_0" # Config 4: F1 F3 F5 (lean)
"0_1_1_0_1_1" # Config 5: F2 F3 F5 F6 (contrast)
)
declare -A CONFIG_DESCRIPTION=(
["1_0_1_1_1_0"]="Heat tracking + Decay + Pipelining metrics + Window inference (minimal overhead)"
["1_0_1_1_1_1"]="Above + Decay slope inference"
["1_1_0_1_1_1"]="Heat + Window history + Pipelining + Both inferences (skip decay loop)"
["1_0_1_0_1_0"]="Heat + Decay + Window inference only (lean config)"
["0_1_1_0_1_1"]="Window + Decay + Both inferences (no heat tracking)"
)
################################################################################
# Configuration Mapping
################################################################################
config_to_loop_flags() {
# Parse config name "L1_L2_L3_L4_L5_L6" and output 6 comma-separated values
local config=$1
# config format: "L1_L2_L3_L4_L5_L6"
local l1=$(echo "$config" | cut -d'_' -f1)
local l2=$(echo "$config" | cut -d'_' -f2)
local l3=$(echo "$config" | cut -d'_' -f3)
local l4=$(echo "$config" | cut -d'_' -f4)
local l5=$(echo "$config" | cut -d'_' -f5)
local l6=$(echo "$config" | cut -d'_' -f6)
echo "${l1},${l2},${l3},${l4},${l5},${l6}"
}
################################################################################
# CSV Functions
################################################################################
init_csv_header() {
{
printf 'timestamp,configuration,run_number,build_status,run_status,'
printf 'total_lookups,cache_hits,cache_hit_percent,bucket_hits,bucket_hit_percent,'
printf 'cache_hit_latency_ns,cache_hit_stddev_ns,bucket_search_latency_ns,bucket_search_stddev_ns,'
printf 'context_predictions_total,context_correct,context_accuracy_percent,'
printf 'rolling_window_width,decay_slope,'
printf 'hot_word_count,stale_word_ratio,avg_word_heat,'
printf 'prefetch_accuracy_percent,prefetch_attempts,prefetch_hits,window_tuning_checks,final_effective_window_size,'
printf 'vm_workload_duration_ns_q48,cpu_temp_delta_c_q48,cpu_freq_delta_mhz_q48,'
printf 'decay_rate_q16,decay_min_interval_ns,rolling_window_size,adaptive_shrink_rate,heat_cache_demotion_threshold,'
printf 'enable_loop_1_heat_tracking,enable_loop_2_rolling_window,enable_loop_3_linear_decay,'
printf 'enable_loop_4_pipelining_metrics,enable_loop_5_window_inference,enable_loop_6_decay_inference,'
printf 'total_heartbeat_ticks,tick_interval_mean_ns,tick_interval_stddev_ns,tick_interval_cv,'
printf 'tick_outlier_count_3sigma,tick_outlier_ratio,'
printf 'cache_hit_percent_rolling_mean,cache_hit_percent_rolling_stddev,cache_stability_index,'
printf 'window_final_effective_size,pattern_diversity_final,diversity_growth_final,window_warmth_achieved,'
printf 'decay_slope_fitted_final,decay_slope_convergence_rate,decay_slope_stable,'
printf 'load_interval_correlation,response_latency_ticks,overshoot_ratio,settling_time_ticks,'
printf 'overall_stability_score\n'
} > "${RESULTS_CSV}"
log_success "CSV header created (with heartbeat metrics)"
}
write_configuration_manifest() {
# Document the 5 elite configurations
{
echo "# StarForth Heartbeat DoE - Top 5 Elite Configurations"
echo "# Generated: $(timestamp)"
echo "# Derived from: Stage 1 baseline (3200 runs, 2^6 factorial)"
echo ""
echo "## Rationale"
echo "These 5 configurations were selected from Stage 1 analysis as exhibiting"
echo "the best trade-offs between performance and stability. This Phase 2 experiment"
echo "adds heartbeat observability to measure temporal stability characteristics:"
echo "- Heartbeat jitter control (CV of tick interval)"
echo "- Convergence speed (decay slope fitting)"
echo "- Load-response coupling (correlation of workload to interval)"
echo "- Overall stability score (composite metric)"
echo ""
echo "## Configuration Format: L1_L2_L3_L4_L5_L6"
echo ""
echo "Factors:"
echo " L1 = ENABLE_LOOP_1_HEAT_TRACKING (execution heat counting)"
echo " L2 = ENABLE_LOOP_2_ROLLING_WINDOW (rolling window history)"
echo " L3 = ENABLE_LOOP_3_LINEAR_DECAY (exponential heat decay)"
echo " L4 = ENABLE_LOOP_4_PIPELINING_METRICS (word transition tracking)"
echo " L5 = ENABLE_LOOP_5_WINDOW_INFERENCE (window width inference)"
echo " L6 = ENABLE_LOOP_6_DECAY_INFERENCE (decay slope inference)"
echo ""
echo "## Elite Configurations (Top 5)"
echo ""
local idx=1
for config in "${TOP5_CONFIGS[@]}"; do
printf "%d. %s\n" $idx "$config"
printf " Description: %s\n" "${CONFIG_DESCRIPTION[$config]}"
echo ""
idx=$((idx + 1))
done
echo "## Analysis Dimensions"
echo ""
echo "### Jitter Control"
echo " - Metric: tick_interval_cv (coefficient of variation)"
echo " - Interpretation: Lower CV = steadier heartbeat = better temporal stability"
echo " - Target: < 0.15 (15% variation is acceptable)"
echo ""
echo "### Convergence Speed"
echo " - Metric: decay_slope_convergence_rate"
echo " - Interpretation: Faster convergence = inference engine tunes quicker"
echo " - Target: Converge within 5000 ticks (5% of typical run)"
echo ""
echo "### Load Response"
echo " - Metric: load_interval_correlation"
echo " - Interpretation: Higher correlation = heartbeat responds smoothly to load"
echo " - Target: > 0.7 (strong coupling, no lag)"
echo ""
echo "### Overall Stability"
echo " - Metric: overall_stability_score (0-100, composite)"
echo " - Components: jitter (weight 1/3) + convergence (1/3) + coupling (1/3)"
echo " - Target: > 75 (excellent stability)"
echo ""
} > "${CONFIG_MANIFEST}"
log_success "Configuration manifest written: ${CONFIG_MANIFEST}"
}
################################################################################
# Build Functions
################################################################################
build_configuration() {
local config_name=$1
local l1=$2
local l2=$3
local l3=$4
local l4=$5
local l5=$6
local l6=$7
cd "${REPO_ROOT}"
# Clean previous build
make clean > /dev/null 2>&1 || true
# Build with specific loop configuration
log_info "Building: make TARGET=${BUILD_PROFILE} with loops: L1=${l1} L2=${l2} L3=${l3} L4=${l4} L5=${l5} L6=${l6}"
if make TARGET="${BUILD_PROFILE}" \
ENABLE_LOOP_1_HEAT_TRACKING="${l1}" \
ENABLE_LOOP_2_ROLLING_WINDOW="${l2}" \
ENABLE_LOOP_3_LINEAR_DECAY="${l3}" \
ENABLE_LOOP_4_PIPELINING_METRICS="${l4}" \
ENABLE_LOOP_5_WINDOW_INFERENCE="${l5}" \
ENABLE_LOOP_6_DECAY_INFERENCE="${l6}" \
> "${LOG_DIR}/build_${config_name}.log" 2>&1; then
log_success "Build completed for ${config_name}"
echo "${BUILD_DIR}/amd64/${BUILD_PROFILE}/starforth"
else
log_error "Build failed for ${config_name}"
cat "${LOG_DIR}/build_${config_name}.log" >&2
return 1
fi
}
################################################################################
# Test Matrix Generation (Randomized)
################################################################################
generate_test_matrix() {
local matrix_file="${TEST_MATRIX}"
> "${matrix_file}"
# Generate all configs × runs
for config in "${TOP5_CONFIGS[@]}"; do
for run in $(seq 1 ${RUNS_PER_CONFIG}); do
echo "${config},${run}" >> "${matrix_file}"
done
done
# Randomize entire matrix
sort -R "${matrix_file}" > "${matrix_file}.shuffled"
mv "${matrix_file}.shuffled" "${matrix_file}"
echo "${matrix_file}"
}
################################################################################
# Execution Functions
################################################################################
run_single_iteration() {
local binary=$1
local config=$2
local run_num=$3
local output_log=$4
if "${binary}" --doe-experiment > "${output_log}" 2>&1; then
return 0
else
return 1
fi
}
run_experiment() {
local matrix_file=$1
log_header "RANDOMIZED HEARTBEAT EXPERIMENT (${TOTAL_RUNS} runs)"
local current_config=""
local current_binary=""
local current_build_status="OK"
local run_index=0
local successful_runs=0
local failed_configs=0
# Read shuffled test matrix
while IFS=',' read -r config_name run_number; do
run_index=$((run_index + 1))
# Build if configuration changed
if [ "${current_config}" != "${config_name}" ]; then
local flags=$(config_to_loop_flags "${config_name}")
local l1="${flags%,*}"
flags="${flags#*,}"
local l2="${flags%,*}"
flags="${flags#*,}"
local l3="${flags%,*}"
flags="${flags#*,}"
local l4="${flags%,*}"
flags="${flags#*,}"
local l5="${flags%,*}"
flags="${flags#*,}"
local l6="${flags#*,}"
# Attempt to build configuration
if current_binary=$(build_configuration "${config_name}" "${l1}" "${l2}" "${l3}" "${l4}" "${l5}" "${l6}" 2>/dev/null); then
current_build_status="OK"
log_success "Build successful for ${config_name}"
else
# Build failed
current_build_status="BUILD_FAILED"
log_error "Configuration ${config_name} failed to build - skipping runs"
current_config="${config_name}"
current_binary=""
continue
fi
current_config="${config_name}"
fi
# Skip execution if build failed
if [ "${current_build_status}" = "BUILD_FAILED" ]; then
log_info "Run ${run_index}/${TOTAL_RUNS} - config ${config_name} #${run_number} skipped (build failed)"
continue
fi
# Execute run
local run_log="${LOG_DIR}/${config_name}_run_${run_number}.log"
local start_time=$(date +%s)
log_info "Run ${run_index}/${TOTAL_RUNS} - config ${config_name} #${run_number}..."
if run_single_iteration "${current_binary}" "${config_name}" "${run_number}" "${run_log}"; then
# Extract CSV row
local csv_row
if ! csv_row=$(extract_final_csv_row "${run_log}"); then
log_error "Run ${run_index}/${TOTAL_RUNS} missing metrics"
continue
fi
if [ -z "${csv_row}" ]; then
log_error "Run ${run_index}/${TOTAL_RUNS} empty metrics"
continue
fi
# Extract heartbeat streams for analysis
local hb_ts="${LOG_DIR}/${config_name}_run_${run_number}_hb_timeseries.csv"
local hb_detail="${LOG_DIR}/${config_name}_run_${run_number}_hb_detail.csv"
extract_heartbeat_timeseries "${run_log}" "${hb_ts}"
extract_heartbeat_detail_csv "${run_log}" "${hb_detail}"
# Append config flags and heartbeat data to CSV row
local flags=$(config_to_loop_flags "${config_name}")
local ts_now=$(timestamp)
printf '%s,%s,%s,%s,%s,%s,%s\n' "${ts_now}" "${config_name}" "${run_number}" "OK" "OK" "${csv_row}" "${flags}" >> "${RESULTS_CSV}"
successful_runs=$((successful_runs + 1))
local elapsed=$(runtime_seconds ${start_time} $(date +%s))
log_success "Run ${run_index}/${TOTAL_RUNS} completed (${elapsed}s)"
else
log_error "Configuration ${config_name} crashed during execution"
fi
done < "${matrix_file}"
log_header "EXPERIMENT COMPLETE"
log_success "Successful runs: ${successful_runs}/${TOTAL_RUNS}"
}
################################################################################
# Main
################################################################################
main() {
local experiment_start=$(date +%s)
local start_time=$(timestamp)
# Calculate totals
TOTAL_CONFIGS=${#TOP5_CONFIGS[@]}
TOTAL_RUNS=$((TOTAL_CONFIGS * RUNS_PER_CONFIG))
log_header "STARFORTH HEARTBEAT DoE - TOP 5 ELITE CONFIGURATIONS"
log_info "Factors: 6 feedback loops (LOOP_1 through LOOP_6)"
log_info "Configurations: 5 elite (selected from Stage 1 baseline)"
log_info "Runs per config: ${RUNS_PER_CONFIG}"
log_info "TOTAL RUNS: ${TOTAL_RUNS}"
log_info "Heartbeat metrics: YES (per-tick jitter, convergence, coupling)"
log_info "Workload: Complete test harness (936+ FORTH tests)"
log_info "Output: ${OUTPUT_DIR}"
# Initialize
init_csv_header
write_configuration_manifest
log_section "Generating randomized test matrix..."
local matrix_file
if ! matrix_file=$(generate_test_matrix); then
log_error "Failed to generate test matrix"
return 1
fi
log_success "Test matrix generated: ${matrix_file}"
# Show preview
echo ""
log_info "Test Matrix Preview (first 10 of ${TOTAL_RUNS} runs):"
head -10 "${matrix_file}" | sed 's/^/ /'
echo " ..."
echo ""
# Wait for confirmation
log_info "Complete test matrix saved: ${matrix_file}"
read -p "Press ENTER to begin execution (Ctrl+C to abort): "
# Execute
if ! run_experiment "${matrix_file}"; then
log_error "Experiment failed"
return 1
fi
# Summary
local experiment_end=$(date +%s)
local total_seconds=$(runtime_seconds ${experiment_start} ${experiment_end})
local end_time=$(timestamp)
log_header "EXPERIMENT COMPLETE"
log_success "All runs completed!"
log_success "Start time: ${start_time}"
log_success "End time: ${end_time}"
log_success "Total runtime: $(format_duration ${total_seconds})"
log_success "Results: ${RESULTS_CSV}"
log_success "Configs: ${CONFIG_MANIFEST}"
log_success "Logs: ${LOG_DIR}/"
echo ""
log_info "CSV Results Preview (first 5 data rows):"
head -6 "${RESULTS_CSV}" | sed 's/^/ /'
echo ""
log_info "Next steps:"
log_info " 1. Download results to analysis repository"
log_info " 2. Run R analysis: Rscript analyze_heartbeat_stability.R"
log_info " 3. Identify golden configuration by stability score"
log_info " 4. Use golden config as baseline for MamaForth/PapaForth"
return 0
}
main "$@"
exit $?