379 lines
14 KiB
Python
379 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
OPP #2 Regression Detection Analysis
|
||
Analyzes window width optimization results_run_01_2025_12_08 against OPP #1 baseline.
|
||
Detects 6 types of optimization regressions as per REGRESSION_DETECTION_FRAMEWORK.md
|
||
"""
|
||
|
||
# StarForth — Steady-State Virtual Machine Runtime
|
||
#
|
||
# Copyright (c) 2023–2025 Robert A. James
|
||
# All rights reserved.
|
||
#
|
||
# This file is part of the StarForth project.
|
||
#
|
||
# Licensed under the StarForth License, Version 1.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
#
|
||
# You may obtain a copy of the License at:
|
||
# https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
|
||
#
|
||
# This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||
# express or implied, including but not limited to the warranties of
|
||
# merchantability, fitness for a particular purpose, and noninfringement.
|
||
#
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
#
|
||
# StarForth — Steady-State Virtual Machine Runtime
|
||
# Copyright (c) 2023–2025 Robert A. James
|
||
# All rights reserved.
|
||
#
|
||
# This file is part of the StarForth project.
|
||
#
|
||
# Licensed under the StarForth License, Version 1.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
#
|
||
# You may obtain a copy of the License at:
|
||
# https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
|
||
#
|
||
# This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||
# express or implied, including but not limited to the warranties of
|
||
# merchantability, fitness for a particular purpose, and noninfringement.
|
||
#
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
#
|
||
|
||
import csv
|
||
import sys
|
||
from collections import defaultdict
|
||
import statistics
|
||
|
||
# OPP #1 Baseline (from previous experiments)
|
||
OPP1_BASELINE_NS = 5947974 # Mean execution time from OPP #1
|
||
|
||
# Regression thresholds (from REGRESSION_DETECTION_FRAMEWORK.md)
|
||
REGRESSION_THRESHOLDS = {
|
||
'time_increase': 5.0, # >5% slower = FAIL
|
||
'variance_increase': 2.0, # >2x variance increase = WARN
|
||
'p99_increase': 8.0, # >8% p99 slower = WARN
|
||
'heat_ratio_change': 3.0, # >3% hot/stale ratio change = WARN
|
||
'cache_hit_drop': 5.0, # >5% cache hit decrease = WARN
|
||
}
|
||
|
||
def load_and_parse_csv(filepath):
|
||
"""Load OPP #2 results_run_01_2025_12_08 CSV and parse metrics"""
|
||
results = defaultdict(list)
|
||
|
||
with open(filepath, 'r') as f:
|
||
reader = csv.DictReader(f)
|
||
for row in reader:
|
||
config = row['configuration']
|
||
|
||
# Parse key metrics (Q48.16 format: divide by 65536)
|
||
metrics = {
|
||
'vm_workload_duration_ns': int(row['vm_workload_duration_ns_q48']) / 65536,
|
||
'cache_hit_percent': float(row['cache_hit_percent']),
|
||
'context_accuracy_percent': float(row['context_accuracy_percent']),
|
||
'prefetch_accuracy_percent': float(row['prefetch_accuracy_percent']),
|
||
'hot_word_count': int(row['hot_word_count']),
|
||
'stale_word_ratio': float(row['stale_word_ratio']),
|
||
}
|
||
|
||
results[config].append(metrics)
|
||
|
||
return results
|
||
|
||
def compute_stats(values):
|
||
"""Compute statistics for a list of values"""
|
||
sorted_vals = sorted(values)
|
||
n = len(sorted_vals)
|
||
|
||
return {
|
||
'mean': statistics.mean(values),
|
||
'stdev': statistics.stdev(values) if n > 1 else 0,
|
||
'min': min(values),
|
||
'max': max(values),
|
||
'p50': sorted_vals[n // 2],
|
||
'p95': sorted_vals[int(n * 0.95)],
|
||
'p99': sorted_vals[int(n * 0.99)] if n >= 100 else sorted_vals[-1],
|
||
'count': n,
|
||
}
|
||
|
||
def detect_metric_inversion(config_stats):
|
||
"""REGRESSION TYPE 1: Metric inversion (better metric gets worse)"""
|
||
regressions = []
|
||
|
||
for config, stats_list in config_stats.items():
|
||
for metric_name in ['cache_hit_percent', 'context_accuracy_percent', 'prefetch_accuracy_percent']:
|
||
values = [m[metric_name] for m in stats_list]
|
||
stat = compute_stats(values)
|
||
|
||
# These metrics should not decrease significantly
|
||
if stat['mean'] < 70.0: # Arbitrary low threshold for quality metrics
|
||
regressions.append({
|
||
'type': 'METRIC_INVERSION',
|
||
'config': config,
|
||
'metric': metric_name,
|
||
'severity': 'MINOR',
|
||
'mean': stat['mean'],
|
||
'detail': f"{metric_name} low at {stat['mean']:.2f}%"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def detect_variance_introduction(config_stats):
|
||
"""REGRESSION TYPE 2: Variance introduction (variance significantly increases)"""
|
||
regressions = []
|
||
|
||
for config, stats_list in config_stats.items():
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
cv = (exec_stats['stdev'] / exec_stats['mean']) * 100 # Coefficient of variation
|
||
|
||
# High variance may indicate instability
|
||
if cv > 20.0: # >20% CV suggests high variability
|
||
regressions.append({
|
||
'type': 'VARIANCE_INTRODUCTION',
|
||
'config': config,
|
||
'metric': 'vm_workload_duration_ns',
|
||
'severity': 'WARN',
|
||
'cv': cv,
|
||
'stdev': exec_stats['stdev'],
|
||
'detail': f"High execution time variance (CV={cv:.2f}%)"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def detect_tail_latency_increase(config_stats):
|
||
"""REGRESSION TYPE 3: Tail latency increase (p99 gets worse)"""
|
||
regressions = []
|
||
|
||
for config, stats_list in config_stats.items():
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
|
||
# Calculate p99 regression vs baseline
|
||
p99_delta_pct = ((exec_stats['p99'] - OPP1_BASELINE_NS) / OPP1_BASELINE_NS) * 100
|
||
|
||
if p99_delta_pct > REGRESSION_THRESHOLDS['p99_increase']:
|
||
regressions.append({
|
||
'type': 'TAIL_LATENCY_INCREASE',
|
||
'config': config,
|
||
'metric': 'p99_latency_ns',
|
||
'severity': 'WARN',
|
||
'p99': exec_stats['p99'],
|
||
'delta_pct': p99_delta_pct,
|
||
'detail': f"P99 latency {p99_delta_pct:.2f}% above baseline"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def detect_mean_time_increase(config_stats):
|
||
"""PRIMARY REGRESSION: Mean execution time increase"""
|
||
regressions = []
|
||
|
||
for config, stats_list in config_stats.items():
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
|
||
# Calculate mean regression vs baseline
|
||
time_delta_pct = ((exec_stats['mean'] - OPP1_BASELINE_NS) / OPP1_BASELINE_NS) * 100
|
||
|
||
if time_delta_pct > REGRESSION_THRESHOLDS['time_increase']:
|
||
regressions.append({
|
||
'type': 'TIME_INCREASE',
|
||
'config': config,
|
||
'severity': 'FAIL',
|
||
'mean': exec_stats['mean'],
|
||
'delta_pct': time_delta_pct,
|
||
'detail': f"Execution time {time_delta_pct:.2f}% above baseline (REGRESSION)"
|
||
})
|
||
elif time_delta_pct > -2.0: # Marginal improvement or no improvement
|
||
regressions.append({
|
||
'type': 'MARGINAL_IMPROVEMENT',
|
||
'config': config,
|
||
'severity': 'INFO',
|
||
'mean': exec_stats['mean'],
|
||
'delta_pct': time_delta_pct,
|
||
'detail': f"Marginal improvement: {time_delta_pct:.2f}% vs baseline"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def detect_cache_hit_drop(config_stats):
|
||
"""REGRESSION TYPE 4: Cache hit rate significant drop"""
|
||
regressions = []
|
||
|
||
for config, stats_list in config_stats.items():
|
||
cache_hits = [m['cache_hit_percent'] for m in stats_list]
|
||
cache_stats = compute_stats(cache_hits)
|
||
|
||
# If cache hit rate is notably low
|
||
if cache_stats['mean'] < 80.0:
|
||
regressions.append({
|
||
'type': 'LOW_CACHE_HIT_RATE',
|
||
'config': config,
|
||
'severity': 'INFO',
|
||
'mean_hit_rate': cache_stats['mean'],
|
||
'detail': f"Cache hit rate {cache_stats['mean']:.2f}% (below 80% target)"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def detect_parameter_interaction(configs_stats):
|
||
"""REGRESSION TYPE 6: Parameter interaction effects"""
|
||
regressions = []
|
||
|
||
# Check if improvement varies significantly across window sizes
|
||
improvements = {}
|
||
for config, stats_list in configs_stats.items():
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
delta_pct = ((exec_stats['mean'] - OPP1_BASELINE_NS) / OPP1_BASELINE_NS) * 100
|
||
improvements[config] = delta_pct
|
||
|
||
if improvements:
|
||
best = max(improvements.values())
|
||
worst = min(improvements.values())
|
||
spread = best - worst
|
||
|
||
# High spread suggests parameter sensitivity
|
||
if spread > 8.0: # >8% spread is significant
|
||
regressions.append({
|
||
'type': 'PARAMETER_SENSITIVITY',
|
||
'severity': 'WARN',
|
||
'improvement_spread_pct': spread,
|
||
'best_config': [k for k, v in improvements.items() if v == best][0],
|
||
'worst_config': [k for k, v in improvements.items() if v == worst][0],
|
||
'detail': f"High sensitivity to window size: {spread:.2f}% spread"
|
||
})
|
||
|
||
return regressions
|
||
|
||
def generate_report(results_csv):
|
||
"""Generate comprehensive regression report"""
|
||
print("=" * 70)
|
||
print("OPP #2 REGRESSION DETECTION ANALYSIS")
|
||
print("=" * 70)
|
||
print()
|
||
|
||
# Load and parse results_run_01_2025_12_08
|
||
config_stats = load_and_parse_csv(results_csv)
|
||
|
||
print(f"OPP #1 Baseline (reference): {OPP1_BASELINE_NS:,.0f} ns")
|
||
print()
|
||
|
||
# Analyze each configuration
|
||
print("CONFIGURATION ANALYSIS:")
|
||
print("-" * 70)
|
||
|
||
for config in sorted(config_stats.keys()):
|
||
stats_list = config_stats[config]
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
|
||
time_delta_pct = ((exec_stats['mean'] - OPP1_BASELINE_NS) / OPP1_BASELINE_NS) * 100
|
||
cv = (exec_stats['stdev'] / exec_stats['mean']) * 100
|
||
|
||
# Extract cache hit average
|
||
cache_hits = [m['cache_hit_percent'] for m in stats_list]
|
||
cache_stats = compute_stats(cache_hits)
|
||
|
||
print(f"\n{config}:")
|
||
print(f" Execution Time:")
|
||
print(f" Mean: {exec_stats['mean']:>12,.0f} ns")
|
||
print(f" Stdev: {exec_stats['stdev']:>12,.0f} ns")
|
||
print(f" CV: {cv:>12.2f}%")
|
||
print(f" Min: {exec_stats['min']:>12,.0f} ns")
|
||
print(f" P50: {exec_stats['p50']:>12,.0f} ns")
|
||
print(f" P95: {exec_stats['p95']:>12,.0f} ns")
|
||
print(f" P99: {exec_stats['p99']:>12,.0f} ns")
|
||
print(f" vs Baseline: {time_delta_pct:>11.2f}%")
|
||
print(f" Cache Hit Rate: {cache_stats['mean']:.2f}%")
|
||
print(f" Samples: {exec_stats['count']} runs")
|
||
|
||
print()
|
||
print("=" * 70)
|
||
print("REGRESSION DETECTION RESULTS:")
|
||
print("=" * 70)
|
||
|
||
# Run all regression detectors
|
||
all_regressions = []
|
||
all_regressions.extend(detect_mean_time_increase(config_stats))
|
||
all_regressions.extend(detect_variance_introduction(config_stats))
|
||
all_regressions.extend(detect_tail_latency_increase(config_stats))
|
||
all_regressions.extend(detect_metric_inversion(config_stats))
|
||
all_regressions.extend(detect_cache_hit_drop(config_stats))
|
||
all_regressions.extend(detect_parameter_interaction(config_stats))
|
||
|
||
# Group by severity
|
||
fails = [r for r in all_regressions if r['severity'] == 'FAIL']
|
||
warns = [r for r in all_regressions if r['severity'] == 'WARN']
|
||
infos = [r for r in all_regressions if r['severity'] in ['INFO', 'MARGINAL_IMPROVEMENT']]
|
||
|
||
if fails:
|
||
print()
|
||
print("FAILURES (Blocking):")
|
||
for reg in fails:
|
||
print(f" ✗ [{reg['config']}] {reg['type']}")
|
||
print(f" {reg['detail']}")
|
||
|
||
if warns:
|
||
print()
|
||
print("WARNINGS (Review Required):")
|
||
for reg in warns:
|
||
config = reg.get('config', 'SYSTEM')
|
||
print(f" ⚠ [{config}] {reg['type']}")
|
||
print(f" {reg['detail']}")
|
||
|
||
if infos:
|
||
print()
|
||
print("INFORMATION:")
|
||
for reg in infos:
|
||
config = reg.get('config', 'SYSTEM')
|
||
symbol = "✓" if "improvement" in reg['type'].lower() else "ℹ"
|
||
print(f" {symbol} [{config}] {reg['type']}")
|
||
print(f" {reg['detail']}")
|
||
|
||
print()
|
||
print("=" * 70)
|
||
|
||
# Determine winner
|
||
print("OPTIMIZATION DECISION:")
|
||
print("-" * 70)
|
||
|
||
config_improvements = {}
|
||
for config, stats_list in config_stats.items():
|
||
exec_times = [m['vm_workload_duration_ns'] for m in stats_list]
|
||
exec_stats = compute_stats(exec_times)
|
||
delta_pct = ((exec_stats['mean'] - OPP1_BASELINE_NS) / OPP1_BASELINE_NS) * 100
|
||
config_improvements[config] = delta_pct
|
||
|
||
best_config = min(config_improvements.items(), key=lambda x: x[1])
|
||
|
||
if best_config[1] > 0:
|
||
print(f"⚠ WARNING: All configurations show regression vs baseline!")
|
||
print(f"Best result: {best_config[0]} with {best_config[1]:+.2f}%")
|
||
else:
|
||
print(f"✓ Best configuration: {best_config[0]}")
|
||
print(f" Improvement: {best_config[1]:.2f}% faster than OPP #1 baseline")
|
||
print()
|
||
print("Recommendation: Lock this configuration for OPP #3")
|
||
|
||
print()
|
||
print(f"Total regressions detected: {len(all_regressions)}")
|
||
print(f" - Failures: {len(fails)}")
|
||
print(f" - Warnings: {len(warns)}")
|
||
print(f" - Informational: {len(infos)}")
|
||
print()
|
||
|
||
if __name__ == '__main__':
|
||
if len(sys.argv) < 2:
|
||
print(f"Usage: {sys.argv[0]} <experiment_results.csv>")
|
||
sys.exit(1)
|
||
|
||
csv_file = sys.argv[1]
|
||
generate_report(csv_file)
|