diff --git a/Tools/AiResponseValidator/.cr/personal/FavoritesList/List.xml b/Tools/AiResponseValidator/.cr/personal/FavoritesList/List.xml new file mode 100644 index 000000000..a60e5ed6c --- /dev/null +++ b/Tools/AiResponseValidator/.cr/personal/FavoritesList/List.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Tools/AiResponseValidator/AiResponseValidator.sln b/Tools/AiResponseValidator/AiResponseValidator.sln new file mode 100644 index 000000000..d85af9a4f --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator.sln @@ -0,0 +1,40 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35707.178 d17.12 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AiResponseValidator", "AiResponseValidator\AiResponseValidator.csproj", "{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AICore", "..\..\Server\Components\AICore\AICore.csproj", "{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServerUtils", "..\..\Server\Components\ServerUtils\ServerUtils.csproj", "{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "..\..\Shared\Shared.csproj", "{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU + {69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.Build.0 = Release|Any CPU + {EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.Build.0 = Release|Any CPU + {2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/Tools/AiResponseValidator/AiResponseValidator/.cr/personal/FavoritesList/List.xml b/Tools/AiResponseValidator/AiResponseValidator/.cr/personal/FavoritesList/List.xml new file mode 100644 index 000000000..a60e5ed6c --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/.cr/personal/FavoritesList/List.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Tools/AiResponseValidator/AiResponseValidator/AiResponseValidator.csproj b/Tools/AiResponseValidator/AiResponseValidator/AiResponseValidator.csproj new file mode 100644 index 000000000..8afa9e859 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/AiResponseValidator.csproj @@ -0,0 +1,51 @@ + + + + WinExe + net48 + true + latest + AiResponseValidator + AiResponseValidator + + true + true + + + + + + + + + Always + + + + + + + + + + + + + + + + + + diff --git a/Tools/AiResponseValidator/AiResponseValidator/App.xaml b/Tools/AiResponseValidator/AiResponseValidator/App.xaml new file mode 100644 index 000000000..e2c796a7f --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/App.xaml @@ -0,0 +1,8 @@ + + + diff --git a/Tools/AiResponseValidator/AiResponseValidator/App.xaml.cs b/Tools/AiResponseValidator/AiResponseValidator/App.xaml.cs new file mode 100644 index 000000000..c4c200e15 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/App.xaml.cs @@ -0,0 +1,11 @@ +using System.Windows; + +namespace AiResponseValidator +{ + /// + /// Interaction logic for App.xaml + /// + public partial class App : Application + { + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/AssemblyInfo.cs b/Tools/AiResponseValidator/AiResponseValidator/AssemblyInfo.cs new file mode 100644 index 000000000..b0ec82757 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/AssemblyInfo.cs @@ -0,0 +1,10 @@ +using System.Windows; + +[assembly: ThemeInfo( + ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located + //(used if a resource is not found in the page, + // or application resource dictionaries) + ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located + //(used if a resource is not found in the page, + // app, or any theme specific resource dictionaries) +)] diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ConsistencyAxis.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ConsistencyAxis.cs new file mode 100644 index 000000000..885d02d9a --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ConsistencyAxis.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; + +namespace AiResponseValidator.Core.Axes +{ + /// + /// Konsistenz: alles fix, nur die Wiederholung läuft (die 'x'-Achse aus dem + /// Juni-Entwurf). Auswertung interessiert die Streuung über die Läufe. + /// + public sealed class ConsistencyAxis : IAxis + { + private readonly int _Runs; + + /// Anzahl Wiederholungen; 0 = verwenden. + public ConsistencyAxis(int runs = 0) + { + _Runs = runs; + } + + public string Name => "Konsistenz"; + + public IEnumerable Expand(TestPlan plan) + { + var runs = _Runs > 0 ? _Runs : System.Math.Max(1, plan.Repeat); + + foreach (var model in plan.Models) + foreach (var request in plan.Requests) + for (var run = 0; run < runs; run++) + { + yield return new TestCase( + RequestId: request.Id, + ModelId: model, + InputVariant: "base", + RunIndex: run, + SystemPrompt: plan.SystemPrompt, + UserMessage: request.UserMessage, + Rule: request.Rule, + Options: plan.Options); + } + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/IAxis.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/IAxis.cs new file mode 100644 index 000000000..dd28a207a --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/IAxis.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; + +namespace AiResponseValidator.Core.Axes +{ + /// + /// Eine Achse expandiert einen zum kartesischen Produkt + /// konkreter s. Der Runner kennt nur die flache Fallliste + /// und weiß nichts von "Robustheit" vs. "Vergleich" – das ist genau der eine, + /// austauschbare Regler. + /// + public interface IAxis + { + string Name { get; } + + IEnumerable Expand(TestPlan plan); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/InputVariant.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/InputVariant.cs new file mode 100644 index 000000000..8dfee5dc7 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/InputVariant.cs @@ -0,0 +1,8 @@ +namespace AiResponseValidator.Core.Axes +{ + /// + /// Eine benannte Eingabevariante für den semantischen Robustheitsmodus – + /// z.B. "widerspruch_1" mit einer bewusst regelverletzenden Nutzeraufforderung. + /// + public sealed record InputVariant(string Label, string UserMessage); +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ModelComparisonAxis.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ModelComparisonAxis.cs new file mode 100644 index 000000000..7a63364f3 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/ModelComparisonAxis.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; + +namespace AiResponseValidator.Core.Axes +{ + /// + /// Modellvergleich: Anfragen-Satz fix, das Modell wird durchgetauscht. + /// Auswertung interessiert Aggregat-Kennzahlen pro Modell (Erfolgsquote, Latenz). + /// + public sealed class ModelComparisonAxis : IAxis + { + public string Name => "Modellvergleich"; + + public IEnumerable Expand(TestPlan plan) + { + foreach (var model in plan.Models) + foreach (var request in plan.Requests) + for (var run = 0; run < System.Math.Max(1, plan.Repeat); run++) + { + yield return new TestCase( + RequestId: request.Id, + ModelId: model, + InputVariant: "base", + RunIndex: run, + SystemPrompt: plan.SystemPrompt, + UserMessage: request.UserMessage, + Rule: request.Rule, + Options: plan.Options); + } + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/PressureMode.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/PressureMode.cs new file mode 100644 index 000000000..9caf01126 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/PressureMode.cs @@ -0,0 +1,20 @@ +namespace AiResponseValidator.Core.Axes +{ + /// + /// Zwei verschiedene Robustheitsbegriffe auf derselben Achse: + /// + public enum PressureMode + { + /// + /// Reiner Kontextfenster-Druck: dieselbe Aufgabe mit zunehmend längerem + /// Füll-/Rauschtext davor (wie buildChunk im AiTokenAnalyzer). + /// + ContextFill, + + /// + /// Semantischer Druck: zunehmend komplexere/widersprüchliche Eingaben, + /// als explizit vorgegebene Varianten. + /// + SemanticPressure + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/RobustnessAxis.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/RobustnessAxis.cs new file mode 100644 index 000000000..14031acb7 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Axes/RobustnessAxis.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace AiResponseValidator.Core.Axes +{ + /// + /// Robustheit: Modell und Regel fix, die Eingabe wächst schrittweise. Interessant + /// ist der Bruchpunkt – ab welcher Eingabegröße/-komplexität die Regel kippt. + /// + /// Der hält die zwei Robustheitsbegriffe sauber + /// auseinander: (Zeichenzahl-Stufen via + /// Füll­text) bzw. (vorgegebene Varianten). + /// + public sealed class RobustnessAxis : IAxis + { + private readonly PressureMode _Mode; + private readonly IReadOnlyList _Sizes; + private readonly string _Filler; + private readonly IReadOnlyList _Variants; + + /// Kontextfenster-Druck: dieselbe Anfrage, davor Füll­text auf N Zeichen. + public static RobustnessAxis ContextFill(IReadOnlyList sizes, string filler) + => new RobustnessAxis(PressureMode.ContextFill, sizes, filler ?? "", null); + + /// Semantischer Druck: explizit vorgegebene, zunehmend härtere Eingaben. + public static RobustnessAxis Semantic(IReadOnlyList variants) + => new RobustnessAxis(PressureMode.SemanticPressure, null, null, variants); + + private RobustnessAxis(PressureMode mode, IReadOnlyList sizes, string filler, IReadOnlyList variants) + { + _Mode = mode; + _Sizes = sizes ?? new int[0]; + _Filler = filler; + _Variants = variants ?? new InputVariant[0]; + } + + public string Name => _Mode == PressureMode.ContextFill ? "Robustheit (Kontext)" : "Robustheit (Semantik)"; + + public IEnumerable Expand(TestPlan plan) + { + var runs = Math.Max(1, plan.Repeat); + + foreach (var model in plan.Models) + foreach (var request in plan.Requests) + foreach (var variant in BuildVariants(request)) + for (var run = 0; run < runs; run++) + { + yield return new TestCase( + RequestId: request.Id, + ModelId: model, + InputVariant: variant.Label, + RunIndex: run, + SystemPrompt: plan.SystemPrompt, + UserMessage: variant.UserMessage, + Rule: request.Rule, + Options: plan.Options); + } + } + + private IEnumerable BuildVariants(RequestSpec request) + { + if (_Mode == PressureMode.SemanticPressure) + return _Variants; + + var list = new List(); + + foreach (var size in _Sizes) + { + var padded = Pad(_Filler, size) + request.UserMessage; + list.Add(new InputVariant($"ctx:{size}", padded)); + } + + return list; + } + + /// Wiederholt/kürzt den Füll­text auf genau Zeichen. + private static string Pad(string filler, int targetChars) + { + if (targetChars <= 0 || string.IsNullOrEmpty(filler)) + return ""; + + var sb = new StringBuilder(targetChars + filler.Length); + + while (sb.Length < targetChars) + sb.Append(filler); + + return sb.ToString(0, targetChars); + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/IsExternalInit.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/IsExternalInit.cs new file mode 100644 index 000000000..00d8f07cb --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/IsExternalInit.cs @@ -0,0 +1,6 @@ +// Ermöglicht C#-9-'record'/'init'-Accessoren unter net48. +// Der Compiler erwartet diesen Typ; im .NET Framework ist er nicht vorhanden. +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit { } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/RequestSpec.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/RequestSpec.cs new file mode 100644 index 000000000..06f341c30 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/RequestSpec.cs @@ -0,0 +1,10 @@ +using AiResponseValidator.Core.Rules; + +namespace AiResponseValidator.Core +{ + /// + /// Eine Anfrage aus dem "Anfragen-Satz" (linke Spalte des Juni-Entwurfs): + /// eine User-Nachricht plus die Regel, die ihre Antwort erfüllen muss. + /// + public sealed record RequestSpec(string Id, string UserMessage, IRule Rule); +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/RuleResult.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/RuleResult.cs new file mode 100644 index 000000000..9252e54b2 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/RuleResult.cs @@ -0,0 +1,15 @@ +namespace AiResponseValidator.Core +{ + /// + /// Ergebnis einer Prüfregel auf eine LLM-Antwort: bestanden oder nicht, + /// samt Regelname und (bei Fehlschlag) einer kurzen Begründung. + /// + public sealed record RuleResult(bool Pass, string RuleName, string Detail) + { + public static RuleResult Ok(string ruleName) + => new RuleResult(true, ruleName, null); + + public static RuleResult Fail(string ruleName, string detail) + => new RuleResult(false, ruleName, detail); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/CanaryRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/CanaryRule.cs new file mode 100644 index 000000000..b5419d2d6 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/CanaryRule.cs @@ -0,0 +1,27 @@ +namespace AiResponseValidator.Core.Rules +{ + /// + /// Canary-Token-Regel für die Truncation-Erkennung: Ein eindeutiges Token wird + /// am Ende von SystemPrompt oder Eingabe platziert und per Anweisung zurückverlangt. + /// Fehlt es in der Antwort, wurde der Kontext vermutlich still abgeschnitten + /// (passt zum prompt_eval_count-Bruch aus dem AiTokenAnalyzer). + /// + public sealed class CanaryRule : IRule + { + private readonly string _Token; + + public CanaryRule(string token) + { + _Token = token ?? ""; + } + + public string Name => "Canary"; + + public RuleResult Check(string response) + { + return (response ?? "").Contains(_Token) + ? RuleResult.Ok(Name) + : RuleResult.Fail(Name, $"Canary-Token '{_Token}' fehlt (mögliche Truncation)"); + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ContainsRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ContainsRule.cs new file mode 100644 index 000000000..dbf2dcee4 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ContainsRule.cs @@ -0,0 +1,34 @@ +using System; + +namespace AiResponseValidator.Core.Rules +{ + /// Antwort muss (oder darf nicht) einen bestimmten Teilstring enthalten. + public sealed class ContainsRule : IRule + { + private readonly string _Needle; + private readonly bool _MustContain; + private readonly bool _IgnoreCase; + + public ContainsRule(string needle, bool mustContain = true, bool ignoreCase = true) + { + _Needle = needle ?? ""; + _MustContain = mustContain; + _IgnoreCase = ignoreCase; + } + + public string Name => _MustContain ? "Contains" : "NotContains"; + + public RuleResult Check(string response) + { + var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var contains = (response ?? "").IndexOf(_Needle, comparison) >= 0; + + if (contains == _MustContain) + return RuleResult.Ok(Name); + + return RuleResult.Fail(Name, _MustContain + ? $"'{_Needle}' fehlt in der Antwort" + : $"'{_Needle}' unerwartet in der Antwort enthalten"); + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/DelegateRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/DelegateRule.cs new file mode 100644 index 000000000..157c90531 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/DelegateRule.cs @@ -0,0 +1,26 @@ +using System; + +namespace AiResponseValidator.Core.Rules +{ + /// + /// Ad-hoc-Regel aus einem Prädikat – nützlich für Sonderfälle, ohne eine eigene + /// Regelklasse anlegen zu müssen. + /// + public sealed class DelegateRule : IRule + { + private readonly Func _Predicate; + private readonly string _FailDetail; + + public DelegateRule(string name, Func predicate, string failDetail = null) + { + Name = name ?? "Delegate"; + _Predicate = predicate ?? throw new ArgumentNullException(nameof(predicate)); + _FailDetail = failDetail ?? "Prädikat nicht erfüllt"; + } + + public string Name { get; } + + public RuleResult Check(string response) + => _Predicate(response ?? "") ? RuleResult.Ok(Name) : RuleResult.Fail(Name, _FailDetail); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ExactRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ExactRule.cs new file mode 100644 index 000000000..c66d8df92 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/ExactRule.cs @@ -0,0 +1,32 @@ +using System; + +namespace AiResponseValidator.Core.Rules +{ + /// Antwort muss exakt dem erwarteten Text entsprechen (getrimmt). + public sealed class ExactRule : IRule + { + private readonly string _Expected; + private readonly bool _IgnoreCase; + + public ExactRule(string expected, bool ignoreCase = false) + { + _Expected = expected ?? ""; + _IgnoreCase = ignoreCase; + } + + public string Name => "Exact"; + + public RuleResult Check(string response) + { + var actual = (response ?? "").Trim(); + var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + return string.Equals(actual, _Expected.Trim(), comparison) + ? RuleResult.Ok(Name) + : RuleResult.Fail(Name, $"Erwartet '{_Expected}', erhalten '{Shorten(actual)}'"); + } + + internal static string Shorten(string s, int max = 60) + => s == null ? "" : (s.Length <= max ? s : s.Substring(0, max) + "…"); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/IRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/IRule.cs new file mode 100644 index 000000000..432772b4f --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/IRule.cs @@ -0,0 +1,17 @@ +namespace AiResponseValidator.Core.Rules +{ + /// + /// Eine austauschbare Prüfregel auf den Text einer LLM-Antwort. + /// Konkrete Regeln: , , + /// , , , + /// . + /// + public interface IRule + { + /// Kurzer, in der Ergebnis-Matrix anzeigbarer Name der Regel. + string Name { get; } + + /// Prüft die Antwort und liefert bestanden/nicht bestanden zurück. + RuleResult Check(string response); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/JsonSchemaRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/JsonSchemaRule.cs new file mode 100644 index 000000000..278e5cbaf --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/JsonSchemaRule.cs @@ -0,0 +1,78 @@ +using System.Collections.Generic; +using System.Linq; + +namespace AiResponseValidator.Core.Rules +{ + /// + /// Leichtgewichtige JSON-Prüfung ohne externe Abhängigkeit: prüft strukturelle + /// Wohlgeformtheit (ausbalancierte Klammern unter Beachtung von Strings/Escapes) + /// und optional das Vorhandensein erwarteter Top-Level-Schlüssel. + /// + /// Bewusst noch keine vollständige JSON-Schema-Validierung – das ist der + /// vorgesehene Erweiterungspunkt (z.B. später via Newtonsoft.Json.Schema). + /// + public sealed class JsonSchemaRule : IRule + { + private readonly IReadOnlyList _RequiredProperties; + + public JsonSchemaRule(params string[] requiredProperties) + { + _RequiredProperties = requiredProperties ?? new string[0]; + } + + public string Name => "Json"; + + public RuleResult Check(string response) + { + var text = (response ?? "").Trim(); + + if (text.Length == 0) + return RuleResult.Fail(Name, "Leere Antwort"); + + if (!IsWellFormed(text)) + return RuleResult.Fail(Name, "Kein wohlgeformtes JSON (unausgeglichene Klammern)"); + + var missing = _RequiredProperties + .Where(p => !text.Contains("\"" + p + "\"")) + .ToList(); + + return missing.Count == 0 + ? RuleResult.Ok(Name) + : RuleResult.Fail(Name, "Fehlende Schlüssel: " + string.Join(", ", missing)); + } + + /// Klammerbalance über { } und [ ], Strings und Escapes werden übersprungen. + private static bool IsWellFormed(string s) + { + var stack = new Stack(); + var in_string = false; + var escaped = false; + + foreach (var c in s) + { + if (in_string) + { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + + continue; + } + + switch (c) + { + case '"': in_string = true; break; + case '{': stack.Push('}'); break; + case '[': stack.Push(']'); break; + case '}': + case ']': + if (stack.Count == 0 || stack.Pop() != c) + return false; + break; + } + } + + return !in_string && stack.Count == 0; + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/RegexRule.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/RegexRule.cs new file mode 100644 index 000000000..a86956bfb --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Rules/RegexRule.cs @@ -0,0 +1,33 @@ +using System.Text.RegularExpressions; + +namespace AiResponseValidator.Core.Rules +{ + /// Antwort muss (oder darf nicht) auf ein Regex-Muster passen. + public sealed class RegexRule : IRule + { + private readonly Regex _Regex; + private readonly bool _MustMatch; + private readonly string _Pattern; + + public RegexRule(string pattern, bool mustMatch = true, RegexOptions options = RegexOptions.Singleline) + { + _Pattern = pattern ?? ""; + _Regex = new Regex(_Pattern, options); + _MustMatch = mustMatch; + } + + public string Name => "Regex"; + + public RuleResult Check(string response) + { + var matches = _Regex.IsMatch(response ?? ""); + + if (matches == _MustMatch) + return RuleResult.Ok(Name); + + return RuleResult.Fail(Name, _MustMatch + ? $"Muster '{_Pattern}' nicht gefunden" + : $"Muster '{_Pattern}' unerwartet gefunden"); + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/RunMetrics.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/RunMetrics.cs new file mode 100644 index 000000000..8d2d519ea --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/RunMetrics.cs @@ -0,0 +1,16 @@ +namespace AiResponseValidator.Core +{ + /// + /// Messwerte eines einzelnen Laufs. ist die vom + /// Backend gemeldete Anzahl tatsächlich verarbeiteter Prompt-Tokens und dient + /// (wie im AiTokenAnalyzer) zur Erkennung stiller Truncation. + /// + public sealed record RunMetrics( + int InputChars, + long? PromptEvalCount, + long? EvalCount, + int? DurationMs) + { + public static readonly RunMetrics Empty = new RunMetrics(0, null, null, null); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/RunOptions.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/RunOptions.cs new file mode 100644 index 000000000..86ae96111 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/RunOptions.cs @@ -0,0 +1,11 @@ +namespace AiResponseValidator.Core +{ + /// + /// Backend-Optionen für einen Lauf. steuert die + /// Kontextfenstergröße (Ollama options.num_ctx). + /// + public sealed record RunOptions(int? NumCtx = null, string CustomerId = null) + { + public static readonly RunOptions Default = new RunOptions(); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/ITestRunner.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/ITestRunner.cs new file mode 100644 index 000000000..9000a701b --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/ITestRunner.cs @@ -0,0 +1,14 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace AiResponseValidator.Core.Runner +{ + /// + /// Führt genau einen aus und wendet dessen Regel an. + /// Der geteilte Kern: run(request, model, options) → response → prüfregel → pass/fail. + /// + public interface ITestRunner + { + Task RunAsync(TestCase testCase, CancellationToken ct); + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/OllamaTestRunner.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/OllamaTestRunner.cs new file mode 100644 index 000000000..d62e65208 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/OllamaTestRunner.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AICore.Facade.LLM; + +namespace AiResponseValidator.Core.Runner +{ + /// + /// gegen den Ollama-Endpunkt (wiederverwendet den + /// aus AICore, wie der AiTokenAnalyzer). + /// + public sealed class OllamaTestRunner : ITestRunner + { + private readonly string _Url; + private readonly string _ApiKey; + + public OllamaTestRunner(string url, string apiKey) + { + _Url = (url ?? "").Trim(); + _ApiKey = (apiKey ?? "").Trim(); + } + + public async Task RunAsync(TestCase testCase, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + var payload = BuildPayload(testCase); + var input_chars = testCase.UserMessage?.Length ?? 0; + + try + { + var result = await Task.Run(() => + { + var client = new OllamaApiClient(_Url, _ApiKey); + var response = client.SendAiMessageRequest( + payload, + out string message, + out string _, + out int? duration, + out long? prompt_eval_count, + out long? eval_count); + + var ok = response != null && response.Success; + var error = ok ? null : (response != null ? response.ToErrorString() : "Keine Antwort vom Backend"); + + return new RawResult(message, duration, prompt_eval_count, eval_count, error); + }, ct); + + var metrics = new RunMetrics(input_chars, result.PromptEvalCount, result.EvalCount, result.DurationMs); + + if (result.Error != null) + return Failed(testCase, metrics, result.Error); + + var rule = testCase.Rule?.Check(result.Message ?? ""); + + return new TestRun( + testCase.RequestId, testCase.ModelId, testCase.InputVariant, testCase.RunIndex, + result.Message, metrics, rule); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return Failed(testCase, new RunMetrics(input_chars, null, null, null), ex.Message); + } + } + + private static TestRun Failed(TestCase c, RunMetrics metrics, string error) + => new TestRun(c.RequestId, c.ModelId, c.InputVariant, c.RunIndex, null, metrics, null, error); + + private static object BuildPayload(TestCase c) + { + var messages = new List(); + + if (!string.IsNullOrEmpty(c.SystemPrompt)) + messages.Add(new { role = "system", content = c.SystemPrompt }); + + messages.Add(new { role = "user", content = c.UserMessage }); + + var payload = new Dictionary + { + ["model"] = c.ModelId, + ["messages"] = messages, + ["requestid"] = Guid.NewGuid().ToString("N") + }; + + if (!string.IsNullOrEmpty(c.Options?.CustomerId)) + payload["customerid"] = c.Options.CustomerId; + + if (c.Options?.NumCtx is int ctx) + payload["options"] = new { num_ctx = ctx }; + + return payload; + } + + private readonly struct RawResult + { + public RawResult(string message, int? durationMs, long? promptEvalCount, long? evalCount, string error) + { + Message = message; + DurationMs = durationMs; + PromptEvalCount = promptEvalCount; + EvalCount = evalCount; + Error = error; + } + + public string Message { get; } + public int? DurationMs { get; } + public long? PromptEvalCount { get; } + public long? EvalCount { get; } + public string Error { get; } + } + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/TestExecutor.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/TestExecutor.cs new file mode 100644 index 000000000..3d45c8183 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/Runner/TestExecutor.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AiResponseValidator.Core.Axes; + +namespace AiResponseValidator.Core.Runner +{ + /// + /// Verbindet Achse, Plan und Runner: expandiert den Plan entlang der gewählten + /// Achse zu Fällen und führt sie (sequenziell) aus. Jeder fertige + /// wird sofort über gemeldet, + /// damit die UI live grün/rot färben kann – der Runner selbst bleibt UI-frei. + /// + public sealed class TestExecutor + { + private readonly ITestRunner _Runner; + + public TestExecutor(ITestRunner runner) + { + _Runner = runner; + } + + public async Task> RunAsync( + TestPlan plan, + IAxis axis, + IProgress onCompleted, + IProgress onProgress, + CancellationToken ct) + { + var cases = axis.Expand(plan).ToList(); + var results = new List(cases.Count); + + for (var i = 0; i < cases.Count; i++) + { + ct.ThrowIfCancellationRequested(); + + onProgress?.Report(new ExecutionProgress(i, cases.Count, cases[i])); + + var run = await _Runner.RunAsync(cases[i], ct); + + results.Add(run); + onCompleted?.Report(run); + } + + return results; + } + } + + /// Fortschrittsmeldung: Fall von startet. + public sealed record ExecutionProgress(int Index, int Total, TestCase Current); +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/TestCase.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/TestCase.cs new file mode 100644 index 000000000..14aa1f53e --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/TestCase.cs @@ -0,0 +1,20 @@ +using AiResponseValidator.Core.Rules; + +namespace AiResponseValidator.Core +{ + /// + /// Eine einzelne, ausführbare Anweisung an den Runner – das Ergebnis der + /// Expansion eines entlang einer . + /// Die vier Identitätsfelder (RequestId, ModelId, InputVariant, RunIndex) sind + /// exakt die Achsen der Ergebnis-Matrix. + /// + public sealed record TestCase( + string RequestId, + string ModelId, + string InputVariant, + int RunIndex, + string SystemPrompt, + string UserMessage, + IRule Rule, + RunOptions Options); +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/TestPlan.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/TestPlan.cs new file mode 100644 index 000000000..fc4e5dbe5 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/TestPlan.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; + +namespace AiResponseValidator.Core +{ + /// + /// Die gemeinsame Konfiguration aller Szenarien. Eine + /// entscheidet, welche dieser Dimensionen durchlaufen wird – der Plan selbst + /// ist szenario-neutral. + /// + public sealed class TestPlan + { + public string SystemPrompt { get; set; } + + /// Der Anfragen-Satz (linke Spalte im Entwurf). + public IReadOnlyList Requests { get; set; } = new List(); + + /// Die zu prüfenden Modelle. + public IReadOnlyList Models { get; set; } = new List(); + + public RunOptions Options { get; set; } = RunOptions.Default; + + /// Wiederholungen pro Fall (die 'x'-Achse aus dem Juni-Entwurf). + public int Repeat { get; set; } = 1; + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/Core/TestRun.cs b/Tools/AiResponseValidator/AiResponseValidator/Core/TestRun.cs new file mode 100644 index 000000000..10ff7b05a --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/Core/TestRun.cs @@ -0,0 +1,25 @@ +namespace AiResponseValidator.Core +{ + /// + /// Das Ergebnis genau eines ausgeführten – die eine, flache + /// Zeile, aus der jede Szenario-Sicht nur eine andere GROUP-BY-Projektion ist. + /// Bewusst unveränderlich, damit sie sich gefahrlos als Roh-Datensatz exportieren + /// und später erneut auswerten lässt (ohne die LLMs neu zu befragen). + /// + public sealed record TestRun( + string RequestId, + string ModelId, + string InputVariant, + int RunIndex, + string Response, + RunMetrics Metrics, + RuleResult Rule, + string Error = null) + { + /// Der Request lief technisch durch (kein Transport-/Backend-Fehler). + public bool Completed => Error == null; + + /// Grün in der Matrix: durchgelaufen UND Regel erfüllt. + public bool Passed => Completed && Rule != null && Rule.Pass; + } +} diff --git a/Tools/AiResponseValidator/AiResponseValidator/MainWindow.xaml b/Tools/AiResponseValidator/AiResponseValidator/MainWindow.xaml new file mode 100644 index 000000000..61f16ab80 --- /dev/null +++ b/Tools/AiResponseValidator/AiResponseValidator/MainWindow.xaml @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + +