This commit is contained in:
2026-07-20 12:45:40 +02:00
parent b14d853da9
commit b1279c5511
40 changed files with 1507 additions and 0 deletions

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Root Type="DevExpress.CodeRush.Foundation.CodePlaces.Options.FavoritesListContainer">
<Options Language="Neutral">
<Groups />
</Options>
</Root>

View File

@@ -0,0 +1,40 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.12.35707.178 d17.12
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AiResponseValidator", "AiResponseValidator\AiResponseValidator.csproj", "{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AICore", "..\..\Server\Components\AICore\AICore.csproj", "{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServerUtils", "..\..\Server\Components\ServerUtils\ServerUtils.csproj", "{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "..\..\Shared\Shared.csproj", "{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.Build.0 = Release|Any CPU
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.Build.0 = Release|Any CPU
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Root Type="DevExpress.CodeRush.Foundation.CodePlaces.Options.FavoritesListContainer">
<Options Language="Neutral">
<Groups />
</Options>
</Root>

View File

@@ -0,0 +1,51 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>net48</TargetFramework>
<UseWPF>true</UseWPF>
<LangVersion>latest</LangVersion>
<RootNamespace>AiResponseValidator</RootNamespace>
<AssemblyName>AiResponseValidator</AssemblyName>
<!-- Erzeugt die Binding-Redirects in der exe.config (net48). -->
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<ItemGroup>
<None Remove="derprozess.txt" />
</ItemGroup>
<ItemGroup>
<Content Include="derprozess.txt">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>
<!--
Unifizierungs-Assemblies müssen im obersten EXE-Projekt explizit referenziert
werden: SDK-Style-Projekte übernehmen NuGet-Assets aus packages.config-Projekten
(AICore etc.) NICHT transitiv. Ohne diese Refs fehlt die DLL im Output und der
Binding-Redirect zeigt auf die falsche Version. Versionen wie im AiTokenAnalyzer.
-->
<ItemGroup>
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="6.1.2" />
<PackageReference Include="System.Buffers" Version="4.6.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\Server\Components\AICore\AICore.csproj" />
<ProjectReference Include="..\..\..\Server\Components\ServerUtils\ServerUtils.csproj" />
<ProjectReference Include="..\..\..\Shared\Shared.csproj" />
</ItemGroup>
<!--
DevExpress (GAC-installiert, v23.2) wird erst mit der PivotGrid-Matrix aktiviert:
<ItemGroup>
<Reference Include="DevExpress.Xpf.Core.v23.2" />
<Reference Include="DevExpress.Xpf.Grid.v23.2" />
<Reference Include="DevExpress.Xpf.PivotGrid.v23.2" />
</ItemGroup>
-->
</Project>

View File

@@ -0,0 +1,8 @@
<Application
x:Class="AiResponseValidator.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="clr-namespace:AiResponseValidator"
StartupUri="MainWindow.xaml">
<Application.Resources />
</Application>

View File

@@ -0,0 +1,11 @@
using System.Windows;
namespace AiResponseValidator
{
/// <summary>
/// Interaction logic for App.xaml
/// </summary>
public partial class App : Application
{
}
}

View File

@@ -0,0 +1,10 @@
using System.Windows;
[assembly: ThemeInfo(
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
//(used if a resource is not found in the page,
// or application resource dictionaries)
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
//(used if a resource is not found in the page,
// app, or any theme specific resource dictionaries)
)]

View File

@@ -0,0 +1,41 @@
using System.Collections.Generic;
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Konsistenz: alles fix, nur die Wiederholung läuft (die 'x'-Achse aus dem
/// Juni-Entwurf). Auswertung interessiert die Streuung über die Läufe.
/// </summary>
public sealed class ConsistencyAxis : IAxis
{
private readonly int _Runs;
/// <param name="runs">Anzahl Wiederholungen; 0 = <see cref="TestPlan.Repeat"/> verwenden.</param>
public ConsistencyAxis(int runs = 0)
{
_Runs = runs;
}
public string Name => "Konsistenz";
public IEnumerable<TestCase> Expand(TestPlan plan)
{
var runs = _Runs > 0 ? _Runs : System.Math.Max(1, plan.Repeat);
foreach (var model in plan.Models)
foreach (var request in plan.Requests)
for (var run = 0; run < runs; run++)
{
yield return new TestCase(
RequestId: request.Id,
ModelId: model,
InputVariant: "base",
RunIndex: run,
SystemPrompt: plan.SystemPrompt,
UserMessage: request.UserMessage,
Rule: request.Rule,
Options: plan.Options);
}
}
}
}

View File

@@ -0,0 +1,17 @@
using System.Collections.Generic;
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Eine Achse expandiert einen <see cref="TestPlan"/> zum kartesischen Produkt
/// konkreter <see cref="TestCase"/>s. Der Runner kennt nur die flache Fallliste
/// und weiß nichts von "Robustheit" vs. "Vergleich" das ist genau der eine,
/// austauschbare Regler.
/// </summary>
public interface IAxis
{
string Name { get; }
IEnumerable<TestCase> Expand(TestPlan plan);
}
}

View File

@@ -0,0 +1,8 @@
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Eine benannte Eingabevariante für den semantischen Robustheitsmodus
/// z.B. "widerspruch_1" mit einer bewusst regelverletzenden Nutzeraufforderung.
/// </summary>
public sealed record InputVariant(string Label, string UserMessage);
}

View File

@@ -0,0 +1,31 @@
using System.Collections.Generic;
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Modellvergleich: Anfragen-Satz fix, das Modell wird durchgetauscht.
/// Auswertung interessiert Aggregat-Kennzahlen pro Modell (Erfolgsquote, Latenz).
/// </summary>
public sealed class ModelComparisonAxis : IAxis
{
public string Name => "Modellvergleich";
public IEnumerable<TestCase> Expand(TestPlan plan)
{
foreach (var model in plan.Models)
foreach (var request in plan.Requests)
for (var run = 0; run < System.Math.Max(1, plan.Repeat); run++)
{
yield return new TestCase(
RequestId: request.Id,
ModelId: model,
InputVariant: "base",
RunIndex: run,
SystemPrompt: plan.SystemPrompt,
UserMessage: request.UserMessage,
Rule: request.Rule,
Options: plan.Options);
}
}
}
}

View File

@@ -0,0 +1,20 @@
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Zwei verschiedene Robustheitsbegriffe auf derselben Achse:
/// </summary>
public enum PressureMode
{
/// <summary>
/// Reiner Kontextfenster-Druck: dieselbe Aufgabe mit zunehmend längerem
/// Füll-/Rauschtext davor (wie <c>buildChunk</c> im AiTokenAnalyzer).
/// </summary>
ContextFill,
/// <summary>
/// Semantischer Druck: zunehmend komplexere/widersprüchliche Eingaben,
/// als explizit vorgegebene Varianten.
/// </summary>
SemanticPressure
}
}

View File

@@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace AiResponseValidator.Core.Axes
{
/// <summary>
/// Robustheit: Modell und Regel fix, die Eingabe wächst schrittweise. Interessant
/// ist der Bruchpunkt ab welcher Eingabegröße/-komplexität die Regel kippt.
///
/// Der <see cref="PressureMode"/> hält die zwei Robustheitsbegriffe sauber
/// auseinander: <see cref="PressureMode.ContextFill"/> (Zeichenzahl-Stufen via
/// Füll­text) bzw. <see cref="PressureMode.SemanticPressure"/> (vorgegebene Varianten).
/// </summary>
public sealed class RobustnessAxis : IAxis
{
private readonly PressureMode _Mode;
private readonly IReadOnlyList<int> _Sizes;
private readonly string _Filler;
private readonly IReadOnlyList<InputVariant> _Variants;
/// <summary>Kontextfenster-Druck: dieselbe Anfrage, davor Füll­text auf N Zeichen.</summary>
public static RobustnessAxis ContextFill(IReadOnlyList<int> sizes, string filler)
=> new RobustnessAxis(PressureMode.ContextFill, sizes, filler ?? "", null);
/// <summary>Semantischer Druck: explizit vorgegebene, zunehmend härtere Eingaben.</summary>
public static RobustnessAxis Semantic(IReadOnlyList<InputVariant> variants)
=> new RobustnessAxis(PressureMode.SemanticPressure, null, null, variants);
private RobustnessAxis(PressureMode mode, IReadOnlyList<int> sizes, string filler, IReadOnlyList<InputVariant> variants)
{
_Mode = mode;
_Sizes = sizes ?? new int[0];
_Filler = filler;
_Variants = variants ?? new InputVariant[0];
}
public string Name => _Mode == PressureMode.ContextFill ? "Robustheit (Kontext)" : "Robustheit (Semantik)";
public IEnumerable<TestCase> Expand(TestPlan plan)
{
var runs = Math.Max(1, plan.Repeat);
foreach (var model in plan.Models)
foreach (var request in plan.Requests)
foreach (var variant in BuildVariants(request))
for (var run = 0; run < runs; run++)
{
yield return new TestCase(
RequestId: request.Id,
ModelId: model,
InputVariant: variant.Label,
RunIndex: run,
SystemPrompt: plan.SystemPrompt,
UserMessage: variant.UserMessage,
Rule: request.Rule,
Options: plan.Options);
}
}
private IEnumerable<InputVariant> BuildVariants(RequestSpec request)
{
if (_Mode == PressureMode.SemanticPressure)
return _Variants;
var list = new List<InputVariant>();
foreach (var size in _Sizes)
{
var padded = Pad(_Filler, size) + request.UserMessage;
list.Add(new InputVariant($"ctx:{size}", padded));
}
return list;
}
/// <summary>Wiederholt/kürzt den Füll­text auf genau <paramref name="targetChars"/> Zeichen.</summary>
private static string Pad(string filler, int targetChars)
{
if (targetChars <= 0 || string.IsNullOrEmpty(filler))
return "";
var sb = new StringBuilder(targetChars + filler.Length);
while (sb.Length < targetChars)
sb.Append(filler);
return sb.ToString(0, targetChars);
}
}
}

View File

@@ -0,0 +1,6 @@
// Ermöglicht C#-9-'record'/'init'-Accessoren unter net48.
// Der Compiler erwartet diesen Typ; im .NET Framework ist er nicht vorhanden.
namespace System.Runtime.CompilerServices
{
internal static class IsExternalInit { }
}

View File

@@ -0,0 +1,10 @@
using AiResponseValidator.Core.Rules;
namespace AiResponseValidator.Core
{
/// <summary>
/// Eine Anfrage aus dem "Anfragen-Satz" (linke Spalte des Juni-Entwurfs):
/// eine User-Nachricht plus die Regel, die ihre Antwort erfüllen muss.
/// </summary>
public sealed record RequestSpec(string Id, string UserMessage, IRule Rule);
}

View File

@@ -0,0 +1,15 @@
namespace AiResponseValidator.Core
{
/// <summary>
/// Ergebnis einer Prüfregel auf eine LLM-Antwort: bestanden oder nicht,
/// samt Regelname und (bei Fehlschlag) einer kurzen Begründung.
/// </summary>
public sealed record RuleResult(bool Pass, string RuleName, string Detail)
{
public static RuleResult Ok(string ruleName)
=> new RuleResult(true, ruleName, null);
public static RuleResult Fail(string ruleName, string detail)
=> new RuleResult(false, ruleName, detail);
}
}

View File

@@ -0,0 +1,27 @@
namespace AiResponseValidator.Core.Rules
{
/// <summary>
/// Canary-Token-Regel für die Truncation-Erkennung: Ein eindeutiges Token wird
/// am Ende von SystemPrompt oder Eingabe platziert und per Anweisung zurückverlangt.
/// Fehlt es in der Antwort, wurde der Kontext vermutlich still abgeschnitten
/// (passt zum <c>prompt_eval_count</c>-Bruch aus dem AiTokenAnalyzer).
/// </summary>
public sealed class CanaryRule : IRule
{
private readonly string _Token;
public CanaryRule(string token)
{
_Token = token ?? "";
}
public string Name => "Canary";
public RuleResult Check(string response)
{
return (response ?? "").Contains(_Token)
? RuleResult.Ok(Name)
: RuleResult.Fail(Name, $"Canary-Token '{_Token}' fehlt (mögliche Truncation)");
}
}
}

View File

@@ -0,0 +1,34 @@
using System;
namespace AiResponseValidator.Core.Rules
{
/// <summary>Antwort muss (oder darf nicht) einen bestimmten Teilstring enthalten.</summary>
public sealed class ContainsRule : IRule
{
private readonly string _Needle;
private readonly bool _MustContain;
private readonly bool _IgnoreCase;
public ContainsRule(string needle, bool mustContain = true, bool ignoreCase = true)
{
_Needle = needle ?? "";
_MustContain = mustContain;
_IgnoreCase = ignoreCase;
}
public string Name => _MustContain ? "Contains" : "NotContains";
public RuleResult Check(string response)
{
var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
var contains = (response ?? "").IndexOf(_Needle, comparison) >= 0;
if (contains == _MustContain)
return RuleResult.Ok(Name);
return RuleResult.Fail(Name, _MustContain
? $"'{_Needle}' fehlt in der Antwort"
: $"'{_Needle}' unerwartet in der Antwort enthalten");
}
}
}

View File

@@ -0,0 +1,26 @@
using System;
namespace AiResponseValidator.Core.Rules
{
/// <summary>
/// Ad-hoc-Regel aus einem Prädikat nützlich für Sonderfälle, ohne eine eigene
/// Regelklasse anlegen zu müssen.
/// </summary>
public sealed class DelegateRule : IRule
{
private readonly Func<string, bool> _Predicate;
private readonly string _FailDetail;
public DelegateRule(string name, Func<string, bool> predicate, string failDetail = null)
{
Name = name ?? "Delegate";
_Predicate = predicate ?? throw new ArgumentNullException(nameof(predicate));
_FailDetail = failDetail ?? "Prädikat nicht erfüllt";
}
public string Name { get; }
public RuleResult Check(string response)
=> _Predicate(response ?? "") ? RuleResult.Ok(Name) : RuleResult.Fail(Name, _FailDetail);
}
}

View File

@@ -0,0 +1,32 @@
using System;
namespace AiResponseValidator.Core.Rules
{
/// <summary>Antwort muss exakt dem erwarteten Text entsprechen (getrimmt).</summary>
public sealed class ExactRule : IRule
{
private readonly string _Expected;
private readonly bool _IgnoreCase;
public ExactRule(string expected, bool ignoreCase = false)
{
_Expected = expected ?? "";
_IgnoreCase = ignoreCase;
}
public string Name => "Exact";
public RuleResult Check(string response)
{
var actual = (response ?? "").Trim();
var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
return string.Equals(actual, _Expected.Trim(), comparison)
? RuleResult.Ok(Name)
: RuleResult.Fail(Name, $"Erwartet '{_Expected}', erhalten '{Shorten(actual)}'");
}
internal static string Shorten(string s, int max = 60)
=> s == null ? "" : (s.Length <= max ? s : s.Substring(0, max) + "…");
}
}

View File

@@ -0,0 +1,17 @@
namespace AiResponseValidator.Core.Rules
{
/// <summary>
/// Eine austauschbare Prüfregel auf den Text einer LLM-Antwort.
/// Konkrete Regeln: <see cref="ExactRule"/>, <see cref="ContainsRule"/>,
/// <see cref="RegexRule"/>, <see cref="CanaryRule"/>, <see cref="JsonSchemaRule"/>,
/// <see cref="DelegateRule"/>.
/// </summary>
public interface IRule
{
/// <summary>Kurzer, in der Ergebnis-Matrix anzeigbarer Name der Regel.</summary>
string Name { get; }
/// <summary>Prüft die Antwort und liefert bestanden/nicht bestanden zurück.</summary>
RuleResult Check(string response);
}
}

View File

@@ -0,0 +1,78 @@
using System.Collections.Generic;
using System.Linq;
namespace AiResponseValidator.Core.Rules
{
/// <summary>
/// Leichtgewichtige JSON-Prüfung ohne externe Abhängigkeit: prüft strukturelle
/// Wohlgeformtheit (ausbalancierte Klammern unter Beachtung von Strings/Escapes)
/// und optional das Vorhandensein erwarteter Top-Level-Schlüssel.
///
/// Bewusst noch keine vollständige JSON-Schema-Validierung das ist der
/// vorgesehene Erweiterungspunkt (z.B. später via Newtonsoft.Json.Schema).
/// </summary>
public sealed class JsonSchemaRule : IRule
{
private readonly IReadOnlyList<string> _RequiredProperties;
public JsonSchemaRule(params string[] requiredProperties)
{
_RequiredProperties = requiredProperties ?? new string[0];
}
public string Name => "Json";
public RuleResult Check(string response)
{
var text = (response ?? "").Trim();
if (text.Length == 0)
return RuleResult.Fail(Name, "Leere Antwort");
if (!IsWellFormed(text))
return RuleResult.Fail(Name, "Kein wohlgeformtes JSON (unausgeglichene Klammern)");
var missing = _RequiredProperties
.Where(p => !text.Contains("\"" + p + "\""))
.ToList();
return missing.Count == 0
? RuleResult.Ok(Name)
: RuleResult.Fail(Name, "Fehlende Schlüssel: " + string.Join(", ", missing));
}
/// <summary>Klammerbalance über { } und [ ], Strings und Escapes werden übersprungen.</summary>
private static bool IsWellFormed(string s)
{
var stack = new Stack<char>();
var in_string = false;
var escaped = false;
foreach (var c in s)
{
if (in_string)
{
if (escaped) escaped = false;
else if (c == '\\') escaped = true;
else if (c == '"') in_string = false;
continue;
}
switch (c)
{
case '"': in_string = true; break;
case '{': stack.Push('}'); break;
case '[': stack.Push(']'); break;
case '}':
case ']':
if (stack.Count == 0 || stack.Pop() != c)
return false;
break;
}
}
return !in_string && stack.Count == 0;
}
}
}

View File

@@ -0,0 +1,33 @@
using System.Text.RegularExpressions;
namespace AiResponseValidator.Core.Rules
{
/// <summary>Antwort muss (oder darf nicht) auf ein Regex-Muster passen.</summary>
public sealed class RegexRule : IRule
{
private readonly Regex _Regex;
private readonly bool _MustMatch;
private readonly string _Pattern;
public RegexRule(string pattern, bool mustMatch = true, RegexOptions options = RegexOptions.Singleline)
{
_Pattern = pattern ?? "";
_Regex = new Regex(_Pattern, options);
_MustMatch = mustMatch;
}
public string Name => "Regex";
public RuleResult Check(string response)
{
var matches = _Regex.IsMatch(response ?? "");
if (matches == _MustMatch)
return RuleResult.Ok(Name);
return RuleResult.Fail(Name, _MustMatch
? $"Muster '{_Pattern}' nicht gefunden"
: $"Muster '{_Pattern}' unerwartet gefunden");
}
}
}

View File

@@ -0,0 +1,16 @@
namespace AiResponseValidator.Core
{
/// <summary>
/// Messwerte eines einzelnen Laufs. <see cref="PromptEvalCount"/> ist die vom
/// Backend gemeldete Anzahl tatsächlich verarbeiteter Prompt-Tokens und dient
/// (wie im AiTokenAnalyzer) zur Erkennung stiller Truncation.
/// </summary>
public sealed record RunMetrics(
int InputChars,
long? PromptEvalCount,
long? EvalCount,
int? DurationMs)
{
public static readonly RunMetrics Empty = new RunMetrics(0, null, null, null);
}
}

View File

@@ -0,0 +1,11 @@
namespace AiResponseValidator.Core
{
/// <summary>
/// Backend-Optionen für einen Lauf. <see cref="NumCtx"/> steuert die
/// Kontextfenstergröße (Ollama <c>options.num_ctx</c>).
/// </summary>
public sealed record RunOptions(int? NumCtx = null, string CustomerId = null)
{
public static readonly RunOptions Default = new RunOptions();
}
}

View File

@@ -0,0 +1,14 @@
using System.Threading;
using System.Threading.Tasks;
namespace AiResponseValidator.Core.Runner
{
/// <summary>
/// Führt genau einen <see cref="TestCase"/> aus und wendet dessen Regel an.
/// Der geteilte Kern: run(request, model, options) → response → prüfregel → pass/fail.
/// </summary>
public interface ITestRunner
{
Task<TestRun> RunAsync(TestCase testCase, CancellationToken ct);
}
}

View File

@@ -0,0 +1,117 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AICore.Facade.LLM;
namespace AiResponseValidator.Core.Runner
{
/// <summary>
/// <see cref="ITestRunner"/> gegen den Ollama-Endpunkt (wiederverwendet den
/// <see cref="OllamaApiClient"/> aus AICore, wie der AiTokenAnalyzer).
/// </summary>
public sealed class OllamaTestRunner : ITestRunner
{
private readonly string _Url;
private readonly string _ApiKey;
public OllamaTestRunner(string url, string apiKey)
{
_Url = (url ?? "").Trim();
_ApiKey = (apiKey ?? "").Trim();
}
public async Task<TestRun> RunAsync(TestCase testCase, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
var payload = BuildPayload(testCase);
var input_chars = testCase.UserMessage?.Length ?? 0;
try
{
var result = await Task.Run(() =>
{
var client = new OllamaApiClient(_Url, _ApiKey);
var response = client.SendAiMessageRequest(
payload,
out string message,
out string _,
out int? duration,
out long? prompt_eval_count,
out long? eval_count);
var ok = response != null && response.Success;
var error = ok ? null : (response != null ? response.ToErrorString() : "Keine Antwort vom Backend");
return new RawResult(message, duration, prompt_eval_count, eval_count, error);
}, ct);
var metrics = new RunMetrics(input_chars, result.PromptEvalCount, result.EvalCount, result.DurationMs);
if (result.Error != null)
return Failed(testCase, metrics, result.Error);
var rule = testCase.Rule?.Check(result.Message ?? "");
return new TestRun(
testCase.RequestId, testCase.ModelId, testCase.InputVariant, testCase.RunIndex,
result.Message, metrics, rule);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
return Failed(testCase, new RunMetrics(input_chars, null, null, null), ex.Message);
}
}
private static TestRun Failed(TestCase c, RunMetrics metrics, string error)
=> new TestRun(c.RequestId, c.ModelId, c.InputVariant, c.RunIndex, null, metrics, null, error);
private static object BuildPayload(TestCase c)
{
var messages = new List<object>();
if (!string.IsNullOrEmpty(c.SystemPrompt))
messages.Add(new { role = "system", content = c.SystemPrompt });
messages.Add(new { role = "user", content = c.UserMessage });
var payload = new Dictionary<string, object>
{
["model"] = c.ModelId,
["messages"] = messages,
["requestid"] = Guid.NewGuid().ToString("N")
};
if (!string.IsNullOrEmpty(c.Options?.CustomerId))
payload["customerid"] = c.Options.CustomerId;
if (c.Options?.NumCtx is int ctx)
payload["options"] = new { num_ctx = ctx };
return payload;
}
private readonly struct RawResult
{
public RawResult(string message, int? durationMs, long? promptEvalCount, long? evalCount, string error)
{
Message = message;
DurationMs = durationMs;
PromptEvalCount = promptEvalCount;
EvalCount = evalCount;
Error = error;
}
public string Message { get; }
public int? DurationMs { get; }
public long? PromptEvalCount { get; }
public long? EvalCount { get; }
public string Error { get; }
}
}
}

View File

@@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AiResponseValidator.Core.Axes;
namespace AiResponseValidator.Core.Runner
{
/// <summary>
/// Verbindet Achse, Plan und Runner: expandiert den Plan entlang der gewählten
/// Achse zu Fällen und führt sie (sequenziell) aus. Jeder fertige
/// <see cref="TestRun"/> wird sofort über <paramref name="onCompleted"/> gemeldet,
/// damit die UI live grün/rot färben kann der Runner selbst bleibt UI-frei.
/// </summary>
public sealed class TestExecutor
{
private readonly ITestRunner _Runner;
public TestExecutor(ITestRunner runner)
{
_Runner = runner;
}
public async Task<IReadOnlyList<TestRun>> RunAsync(
TestPlan plan,
IAxis axis,
IProgress<TestRun> onCompleted,
IProgress<ExecutionProgress> onProgress,
CancellationToken ct)
{
var cases = axis.Expand(plan).ToList();
var results = new List<TestRun>(cases.Count);
for (var i = 0; i < cases.Count; i++)
{
ct.ThrowIfCancellationRequested();
onProgress?.Report(new ExecutionProgress(i, cases.Count, cases[i]));
var run = await _Runner.RunAsync(cases[i], ct);
results.Add(run);
onCompleted?.Report(run);
}
return results;
}
}
/// <summary>Fortschrittsmeldung: Fall <see cref="Index"/> von <see cref="Total"/> startet.</summary>
public sealed record ExecutionProgress(int Index, int Total, TestCase Current);
}

View File

@@ -0,0 +1,20 @@
using AiResponseValidator.Core.Rules;
namespace AiResponseValidator.Core
{
/// <summary>
/// Eine einzelne, ausführbare Anweisung an den Runner das Ergebnis der
/// Expansion eines <see cref="TestPlan"/> entlang einer <see cref="Axes.IAxis"/>.
/// Die vier Identitätsfelder (RequestId, ModelId, InputVariant, RunIndex) sind
/// exakt die Achsen der Ergebnis-Matrix.
/// </summary>
public sealed record TestCase(
string RequestId,
string ModelId,
string InputVariant,
int RunIndex,
string SystemPrompt,
string UserMessage,
IRule Rule,
RunOptions Options);
}

View File

@@ -0,0 +1,25 @@
using System.Collections.Generic;
namespace AiResponseValidator.Core
{
/// <summary>
/// Die gemeinsame Konfiguration aller Szenarien. Eine <see cref="Axes.IAxis"/>
/// entscheidet, welche dieser Dimensionen durchlaufen wird der Plan selbst
/// ist szenario-neutral.
/// </summary>
public sealed class TestPlan
{
public string SystemPrompt { get; set; }
/// <summary>Der Anfragen-Satz (linke Spalte im Entwurf).</summary>
public IReadOnlyList<RequestSpec> Requests { get; set; } = new List<RequestSpec>();
/// <summary>Die zu prüfenden Modelle.</summary>
public IReadOnlyList<string> Models { get; set; } = new List<string>();
public RunOptions Options { get; set; } = RunOptions.Default;
/// <summary>Wiederholungen pro Fall (die 'x'-Achse aus dem Juni-Entwurf).</summary>
public int Repeat { get; set; } = 1;
}
}

View File

@@ -0,0 +1,25 @@
namespace AiResponseValidator.Core
{
/// <summary>
/// Das Ergebnis genau eines ausgeführten <see cref="TestCase"/> die eine, flache
/// Zeile, aus der jede Szenario-Sicht nur eine andere GROUP-BY-Projektion ist.
/// Bewusst unveränderlich, damit sie sich gefahrlos als Roh-Datensatz exportieren
/// und später erneut auswerten lässt (ohne die LLMs neu zu befragen).
/// </summary>
public sealed record TestRun(
string RequestId,
string ModelId,
string InputVariant,
int RunIndex,
string Response,
RunMetrics Metrics,
RuleResult Rule,
string Error = null)
{
/// <summary>Der Request lief technisch durch (kein Transport-/Backend-Fehler).</summary>
public bool Completed => Error == null;
/// <summary>Grün in der Matrix: durchgelaufen UND Regel erfüllt.</summary>
public bool Passed => Completed && Rule != null && Rule.Pass;
}
}

View File

@@ -0,0 +1,119 @@
<Window
x:Class="AiResponseValidator.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="AI Response Validator" Height="720" Width="1180">
<DockPanel>
<!-- Toolbar -->
<ToolBarTray DockPanel.Dock="Top">
<ToolBar>
<TextBlock Text="URL:" VerticalAlignment="Center" Margin="4,0" />
<TextBox Text="{Binding Url, UpdateSourceTrigger=PropertyChanged}" Width="200" />
<TextBlock Text="API-Key:" VerticalAlignment="Center" Margin="8,0,4,0" />
<TextBox Text="{Binding ApiKey, UpdateSourceTrigger=PropertyChanged}" Width="120" />
<Separator />
<TextBlock Text="Szenario:" VerticalAlignment="Center" Margin="4,0" />
<ComboBox ItemsSource="{Binding Scenarios}" SelectedItem="{Binding Scenario}" Width="150" />
<Separator />
<Button Content="▶ Ausführen" Command="{Binding RunCommand}" Padding="8,2" />
<Button Content="■ Abbrechen" Command="{Binding CancelCommand}" Padding="8,2" Margin="4,0,0,0" />
</ToolBar>
</ToolBarTray>
<!-- Status -->
<StatusBar DockPanel.Dock="Bottom">
<StatusBarItem>
<TextBlock Text="{Binding StatusText}" />
</StatusBarItem>
</StatusBar>
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="360" />
<ColumnDefinition Width="4" />
<ColumnDefinition Width="*" />
</Grid.ColumnDefinitions>
<!-- Links: Konfiguration / Anfragen-Satz (Grundgerüst: eine Anfrage) -->
<ScrollViewer Grid.Column="0" VerticalScrollBarVisibility="Auto">
<StackPanel Margin="8">
<TextBlock Text="System-Prompt" FontWeight="Bold" />
<TextBox Text="{Binding SystemPrompt, UpdateSourceTrigger=PropertyChanged}"
AcceptsReturn="True" TextWrapping="Wrap" Height="120"
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
<TextBlock Text="Modelle (Komma/Zeile getrennt)" FontWeight="Bold" />
<TextBox Text="{Binding ModelsText, UpdateSourceTrigger=PropertyChanged}"
AcceptsReturn="True" TextWrapping="Wrap" Height="60"
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
<TextBlock Text="Anfrage (User-Nachricht)" FontWeight="Bold" />
<TextBox Text="{Binding RequestMessage, UpdateSourceTrigger=PropertyChanged}"
AcceptsReturn="True" TextWrapping="Wrap" Height="100"
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
<TextBlock Text="Regel" FontWeight="Bold" />
<ComboBox ItemsSource="{Binding RuleKinds}" SelectedItem="{Binding Rule}" Margin="0,2,0,4" />
<TextBox Text="{Binding RuleArgument, UpdateSourceTrigger=PropertyChanged}"
TextWrapping="Wrap" Margin="0,0,0,8"
ToolTip="Contains/Exact/Regex: Text/Muster · Canary: Token · Json: erwartete Schlüssel" />
<TextBlock Text="Größen (nur Robustheit/Kontext)" FontWeight="Bold" />
<TextBox Text="{Binding SizesText, UpdateSourceTrigger=PropertyChanged}" Margin="0,2,0,8" />
<Grid Margin="0,0,0,8">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*" />
<ColumnDefinition Width="*" />
</Grid.ColumnDefinitions>
<StackPanel Grid.Column="0" Margin="0,0,4,0">
<TextBlock Text="Wiederholungen" FontWeight="Bold" />
<TextBox Text="{Binding Repeat, UpdateSourceTrigger=PropertyChanged}" />
</StackPanel>
<StackPanel Grid.Column="1" Margin="4,0,0,0">
<TextBlock Text="num_ctx (optional)" FontWeight="Bold" />
<TextBox Text="{Binding NumCtxText, UpdateSourceTrigger=PropertyChanged}" />
</StackPanel>
</Grid>
</StackPanel>
</ScrollViewer>
<GridSplitter Grid.Column="1" Width="4" HorizontalAlignment="Stretch" Background="#DDD" />
<!-- Mitte: Ergebnis-Matrix (Grundgerüst: flache Liste; PivotGrid folgt) -->
<DataGrid Grid.Column="2" ItemsSource="{Binding Results}" AutoGenerateColumns="False"
IsReadOnly="True" CanUserAddRows="False" Margin="4">
<DataGrid.RowStyle>
<Style TargetType="DataGridRow">
<Style.Triggers>
<DataTrigger Binding="{Binding Passed}" Value="True">
<Setter Property="Background" Value="#D6F5D6" />
</DataTrigger>
<DataTrigger Binding="{Binding Passed}" Value="False">
<Setter Property="Background" Value="#F8D6D6" />
</DataTrigger>
<DataTrigger Binding="{Binding Completed}" Value="False">
<Setter Property="Background" Value="#FBE6C2" />
</DataTrigger>
</Style.Triggers>
</Style>
</DataGrid.RowStyle>
<DataGrid.Columns>
<DataGridTextColumn Header="Modell" Binding="{Binding ModelId}" Width="160" />
<DataGridTextColumn Header="Variante" Binding="{Binding InputVariant}" Width="90" />
<DataGridTextColumn Header="Lauf" Binding="{Binding RunIndex}" Width="45" />
<DataGridTextColumn Header="Regel" Binding="{Binding Rule.Name}" Width="80" />
<DataGridTextColumn Header="Pass" Binding="{Binding Passed}" Width="50" />
<DataGridTextColumn Header="Detail" Binding="{Binding Rule.Detail}" Width="180" />
<DataGridTextColumn Header="prompt_eval" Binding="{Binding Metrics.PromptEvalCount}" Width="80" />
<DataGridTextColumn Header="ms" Binding="{Binding Metrics.DurationMs}" Width="60" />
<DataGridTextColumn Header="Antwort" Binding="{Binding Response}" Width="*" />
<DataGridTextColumn Header="Fehler" Binding="{Binding Error}" Width="160" />
</DataGrid.Columns>
</DataGrid>
</Grid>
</DockPanel>
</Window>

View File

@@ -0,0 +1,30 @@
using System.ComponentModel;
using System.Windows;
using AiResponseValidator.ViewModels;
namespace AiResponseValidator
{
/// <summary>
/// Interaction logic for MainWindow.xaml. Setzt den <see cref="MainViewModel"/>
/// als DataContext und lädt/speichert dessen Eingaben beim Start/Schließen.
/// </summary>
public partial class MainWindow : Window
{
private readonly MainViewModel _ViewModel;
public MainWindow()
{
InitializeComponent();
_ViewModel = new MainViewModel();
_ViewModel.LoadSettings();
DataContext = _ViewModel;
}
protected override void OnClosing(CancelEventArgs e)
{
_ViewModel.SaveSettings();
base.OnClosing(e);
}
}
}

View File

@@ -0,0 +1,24 @@
using AiResponseValidator.ViewModels;
namespace AiResponseValidator
{
/// <summary>
/// Beim Schließen gespeicherte / beim Öffnen geladene Eingabewerte
/// (wie im AiTokenAnalyzer, via XmlSerializer in %AppData%).
/// </summary>
public class UserSettings
{
public string Url { get; set; }
public string ApiKey { get; set; }
public string SystemPrompt { get; set; }
public string ModelsText { get; set; }
public string RequestMessage { get; set; }
public RuleKind Rule { get; set; }
public string RuleArgument { get; set; }
public ScenarioKind Scenario { get; set; }
public string SizesText { get; set; }
public string FillerText { get; set; }
public int Repeat { get; set; } = 1;
public string NumCtxText { get; set; }
}
}

View File

@@ -0,0 +1,44 @@
using System;
using System.Threading.Tasks;
using System.Windows.Input;
namespace AiResponseValidator.ViewModels
{
/// <summary>ICommand für asynchrone Aktionen; sperrt sich selbst während der Ausführung.</summary>
public sealed class AsyncRelayCommand : ICommand
{
private readonly Func<Task> _ExecuteAsync;
private readonly Func<bool> _CanExecute;
private bool _IsRunning;
public AsyncRelayCommand(Func<Task> executeAsync, Func<bool> canExecute = null)
{
_ExecuteAsync = executeAsync ?? throw new ArgumentNullException(nameof(executeAsync));
_CanExecute = canExecute;
}
public bool CanExecute(object parameter) => !_IsRunning && (_CanExecute == null || _CanExecute());
public async void Execute(object parameter)
{
_IsRunning = true;
CommandManager.InvalidateRequerySuggested();
try
{
await _ExecuteAsync();
}
finally
{
_IsRunning = false;
CommandManager.InvalidateRequerySuggested();
}
}
public event EventHandler CanExecuteChanged
{
add { CommandManager.RequerySuggested += value; }
remove { CommandManager.RequerySuggested -= value; }
}
}
}

View File

@@ -0,0 +1,316 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Serialization;
using AiResponseValidator.Core;
using AiResponseValidator.Core.Axes;
using AiResponseValidator.Core.Rules;
using AiResponseValidator.Core.Runner;
namespace AiResponseValidator.ViewModels
{
/// <summary>
/// Bindeglied zwischen Toolbar-Konfiguration, Achsenwahl und der flachen
/// Ergebnis-Liste. Hält KEINE Backend-Logik die liegt komplett im Core.
///
/// Erstes Grundgerüst: ein einzelner Anfrage-Editor. Der volle Anfragen-Satz
/// (mehrere <see cref="RequestSpec"/>) sowie die PivotGrid-Matrix folgen als
/// nächster Schritt.
/// </summary>
public sealed class MainViewModel : ViewModelBase
{
private string _Url = "http://localhost:11434";
private string _ApiKey = "";
private string _SystemPrompt = "";
private string _ModelsText = "gemma3_12b_max_context:latest";
private string _RequestMessage = "";
private RuleKind _Rule = RuleKind.Contains;
private string _RuleArgument = "";
private ScenarioKind _Scenario = ScenarioKind.Modellvergleich;
private string _SizesText = "50k, 100k, 150k";
private string _FillerText = "";
private int _Repeat = 1;
private string _NumCtxText = "";
private string _StatusText = "Bereit.";
private CancellationTokenSource _Cts;
public MainViewModel()
{
RunCommand = new AsyncRelayCommand(RunAsync, () => !IsRunning);
CancelCommand = new RelayCommand(Cancel, () => IsRunning);
try
{
_FillerText = File.ReadAllText("derprozess.txt");
}
catch (Exception ex)
{
_FillerText = "Der Affe ist gross. ";
}
}
public ObservableCollection<TestRun> Results { get; } = new ObservableCollection<TestRun>();
public string Url { get => _Url; set => Set(ref _Url, value); }
public string ApiKey { get => _ApiKey; set => Set(ref _ApiKey, value); }
public string SystemPrompt { get => _SystemPrompt; set => Set(ref _SystemPrompt, value); }
public string ModelsText { get => _ModelsText; set => Set(ref _ModelsText, value); }
public string RequestMessage { get => _RequestMessage; set => Set(ref _RequestMessage, value); }
public RuleKind Rule { get => _Rule; set => Set(ref _Rule, value); }
public string RuleArgument { get => _RuleArgument; set => Set(ref _RuleArgument, value); }
public ScenarioKind Scenario { get => _Scenario; set => Set(ref _Scenario, value); }
public string SizesText { get => _SizesText; set => Set(ref _SizesText, value); }
public string FillerText { get => _FillerText; set => Set(ref _FillerText, value); }
public int Repeat { get => _Repeat; set => Set(ref _Repeat, value); }
public string NumCtxText { get => _NumCtxText; set => Set(ref _NumCtxText, value); }
public string StatusText { get => _StatusText; set => Set(ref _StatusText, value); }
private bool _IsRunning;
public bool IsRunning
{
get => _IsRunning;
private set => Set(ref _IsRunning, value);
}
public AsyncRelayCommand RunCommand { get; }
public RelayCommand CancelCommand { get; }
public IEnumerable<ScenarioKind> Scenarios => Enum.GetValues(typeof(ScenarioKind)).Cast<ScenarioKind>();
public IEnumerable<RuleKind> RuleKinds => Enum.GetValues(typeof(RuleKind)).Cast<RuleKind>();
private static string GetSettingsPath()
=> Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "AiResponseValidator", "settings.xml");
/// <summary>Lädt die zuletzt gespeicherten Eingaben; defekte/fehlende Datei = Defaults.</summary>
public void LoadSettings()
{
try
{
var path = GetSettingsPath();
if (!File.Exists(path))
return;
UserSettings s;
using (var stream = File.OpenRead(path))
s = (UserSettings)new XmlSerializer(typeof(UserSettings)).Deserialize(stream);
if (s == null)
return;
Url = s.Url ?? Url;
ApiKey = s.ApiKey ?? ApiKey;
SystemPrompt = s.SystemPrompt ?? SystemPrompt;
ModelsText = s.ModelsText ?? ModelsText;
RequestMessage = s.RequestMessage ?? RequestMessage;
Rule = s.Rule;
RuleArgument = s.RuleArgument ?? RuleArgument;
Scenario = s.Scenario;
SizesText = s.SizesText ?? SizesText;
FillerText = s.FillerText ?? FillerText;
Repeat = s.Repeat > 0 ? s.Repeat : Repeat;
NumCtxText = s.NumCtxText ?? NumCtxText;
}
catch
{
// Defekte Settings-Datei ignorieren, Defaults verwenden.
}
}
/// <summary>Speichert die aktuellen Eingaben; Fehler dürfen das Schließen nicht verhindern.</summary>
public void SaveSettings()
{
try
{
var s = new UserSettings
{
Url = Url,
ApiKey = ApiKey,
SystemPrompt = SystemPrompt,
ModelsText = ModelsText,
RequestMessage = RequestMessage,
Rule = Rule,
RuleArgument = RuleArgument,
Scenario = Scenario,
SizesText = SizesText,
FillerText = FillerText,
Repeat = Repeat,
NumCtxText = NumCtxText
};
var path = GetSettingsPath();
Directory.CreateDirectory(Path.GetDirectoryName(path));
using (var stream = File.Create(path))
new XmlSerializer(typeof(UserSettings)).Serialize(stream, s);
}
catch
{
// Speichern der Settings darf das Schließen nicht verhindern.
}
}
private void Cancel()
{
_Cts?.Cancel();
StatusText = "Abbruch angefordert...";
}
private async Task RunAsync()
{
var models = SplitList(ModelsText);
if (models.Count == 0)
{
StatusText = "Bitte mindestens ein Modell angeben.";
return;
}
if (string.IsNullOrWhiteSpace(RequestMessage))
{
StatusText = "Bitte eine Anfrage (User-Nachricht) angeben.";
return;
}
var plan = BuildPlan(models);
IAxis axis;
try
{
axis = BuildAxis();
}
catch (FormatException ex)
{
StatusText = ex.Message;
return;
}
Results.Clear();
IsRunning = true;
_Cts = new CancellationTokenSource();
var executor = new TestExecutor(new OllamaTestRunner(Url, ApiKey));
var onCompleted = new Progress<TestRun>(r => Results.Add(r));
var onProgress = new Progress<ExecutionProgress>(p =>
StatusText = $"[{axis.Name}] Fall {p.Index + 1}/{p.Total}: {p.Current.ModelId}, Variante '{p.Current.InputVariant}'...");
try
{
var all = await executor.RunAsync(plan, axis, onCompleted, onProgress, _Cts.Token);
var passed = all.Count(r => r.Passed);
StatusText = $"Fertig: {passed}/{all.Count} bestanden.";
}
catch (OperationCanceledException)
{
StatusText = $"Abgebrochen ({Results.Count} Läufe erfasst).";
}
catch (Exception ex)
{
StatusText = "Fehler: " + ex.Message;
}
finally
{
_Cts.Dispose();
_Cts = null;
IsRunning = false;
}
}
private TestPlan BuildPlan(IReadOnlyList<string> models)
{
int? num_ctx = null;
if (int.TryParse((NumCtxText ?? "").Trim(), out var ctx) && ctx > 0)
num_ctx = ctx;
var request = new RequestSpec("req1", RequestMessage, BuildRule());
return new TestPlan
{
SystemPrompt = SystemPrompt,
Requests = new[] { request },
Models = models,
Options = new RunOptions(num_ctx),
Repeat = Math.Max(1, Repeat)
};
}
private IRule BuildRule()
{
var arg = RuleArgument ?? "";
switch (Rule)
{
case RuleKind.Contains: return new ContainsRule(arg, mustContain: true);
case RuleKind.NotContains: return new ContainsRule(arg, mustContain: false);
case RuleKind.Exact: return new ExactRule(arg);
case RuleKind.Regex: return new RegexRule(arg);
case RuleKind.Canary: return new CanaryRule(arg);
case RuleKind.Json: return new JsonSchemaRule(SplitList(arg).ToArray());
default: return new ContainsRule(arg);
}
}
private IAxis BuildAxis()
{
switch (Scenario)
{
case ScenarioKind.Modellvergleich:
return new ModelComparisonAxis();
case ScenarioKind.Konsistenz:
return new ConsistencyAxis();
case ScenarioKind.RobustheitKontext:
return RobustnessAxis.ContextFill(ParseSizes(SizesText), FillerText);
case ScenarioKind.RobustheitSemantik:
// Grundgerüst: eine Variante aus der Anfrage; die Varianten-Liste
// bekommt später einen eigenen Editor.
return RobustnessAxis.Semantic(new[] { new InputVariant("v1", RequestMessage) });
default:
return new ModelComparisonAxis();
}
}
private static List<string> SplitList(string text)
=> (text ?? "")
.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => s.Length > 0)
.ToList();
/// <summary>Parst z.B. "50k, 100k, 150k" zu Zeichenanzahlen (wie im AiTokenAnalyzer).</summary>
private static List<int> ParseSizes(string input)
{
var result = new List<int>();
foreach (var part in SplitList(input))
{
var s = part.ToLowerInvariant().Replace(".", "").Replace(" ", "");
var factor = 1;
if (s.EndsWith("k")) { factor = 1000; s = s.Substring(0, s.Length - 1); }
else if (s.EndsWith("m")) { factor = 1000000; s = s.Substring(0, s.Length - 1); }
if (!int.TryParse(s, out var value) || value <= 0)
throw new FormatException($"Ungültige Zeichenanzahl: '{part}'");
result.Add(value * factor);
}
if (result.Count == 0)
throw new FormatException("Bitte mindestens eine Zeichenanzahl angeben (z.B. '50k, 100k').");
return result;
}
}
}

View File

@@ -0,0 +1,28 @@
using System;
using System.Windows.Input;
namespace AiResponseValidator.ViewModels
{
/// <summary>Einfaches ICommand für synchrone Aktionen.</summary>
public sealed class RelayCommand : ICommand
{
private readonly Action _Execute;
private readonly Func<bool> _CanExecute;
public RelayCommand(Action execute, Func<bool> canExecute = null)
{
_Execute = execute ?? throw new ArgumentNullException(nameof(execute));
_CanExecute = canExecute;
}
public bool CanExecute(object parameter) => _CanExecute == null || _CanExecute();
public void Execute(object parameter) => _Execute();
public event EventHandler CanExecuteChanged
{
add { CommandManager.RequerySuggested += value; }
remove { CommandManager.RequerySuggested -= value; }
}
}
}

View File

@@ -0,0 +1,22 @@
namespace AiResponseValidator.ViewModels
{
/// <summary>Die vom Toolbar-Regler wählbaren Szenarien (je eine Achse).</summary>
public enum ScenarioKind
{
RobustheitKontext,
RobustheitSemantik,
Konsistenz,
Modellvergleich
}
/// <summary>Regeltyp für den einfachen Ein-Anfrage-Editor des Grundgerüsts.</summary>
public enum RuleKind
{
Contains,
NotContains,
Exact,
Regex,
Canary,
Json
}
}

View File

@@ -0,0 +1,24 @@
using System.ComponentModel;
using System.Runtime.CompilerServices;
namespace AiResponseValidator.ViewModels
{
/// <summary>Minimale MVVM-Basis (manuelles INotifyPropertyChanged, wie im AiTokenAnalyzer).</summary>
public abstract class ViewModelBase : INotifyPropertyChanged
{
public event PropertyChangedEventHandler PropertyChanged;
protected bool Set<T>(ref T field, T value, [CallerMemberName] string name = null)
{
if (Equals(field, value))
return false;
field = value;
Raise(name);
return true;
}
protected void Raise([CallerMemberName] string name = null)
=> PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name));
}
}

File diff suppressed because one or more lines are too long