update 1
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Root Type="DevExpress.CodeRush.Foundation.CodePlaces.Options.FavoritesListContainer">
|
||||
<Options Language="Neutral">
|
||||
<Groups />
|
||||
</Options>
|
||||
</Root>
|
||||
40
Tools/AiResponseValidator/AiResponseValidator.sln
Normal file
40
Tools/AiResponseValidator/AiResponseValidator.sln
Normal file
@@ -0,0 +1,40 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.12.35707.178 d17.12
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AiResponseValidator", "AiResponseValidator\AiResponseValidator.csproj", "{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AICore", "..\..\Server\Components\AICore\AICore.csproj", "{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServerUtils", "..\..\Server\Components\ServerUtils\ServerUtils.csproj", "{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "..\..\Shared\Shared.csproj", "{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{7B3E1A2C-4D5F-4A6B-9C8D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{69D70CA9-0DF9-419E-BFAF-F4539F129BD9}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{EC2349FB-7FE0-4AD1-B28B-A7A27AF80A57}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2F50B83D-A3F0-4EC4-979A-3F9B7E3D8ED4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Root Type="DevExpress.CodeRush.Foundation.CodePlaces.Options.FavoritesListContainer">
|
||||
<Options Language="Neutral">
|
||||
<Groups />
|
||||
</Options>
|
||||
</Root>
|
||||
@@ -0,0 +1,51 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>WinExe</OutputType>
|
||||
<TargetFramework>net48</TargetFramework>
|
||||
<UseWPF>true</UseWPF>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<RootNamespace>AiResponseValidator</RootNamespace>
|
||||
<AssemblyName>AiResponseValidator</AssemblyName>
|
||||
<!-- Erzeugt die Binding-Redirects in der exe.config (net48). -->
|
||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="derprozess.txt" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="derprozess.txt">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
<!--
|
||||
Unifizierungs-Assemblies müssen im obersten EXE-Projekt explizit referenziert
|
||||
werden: SDK-Style-Projekte übernehmen NuGet-Assets aus packages.config-Projekten
|
||||
(AICore etc.) NICHT transitiv. Ohne diese Refs fehlt die DLL im Output und der
|
||||
Binding-Redirect zeigt auf die falsche Version. Versionen wie im AiTokenAnalyzer.
|
||||
-->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="6.1.2" />
|
||||
<PackageReference Include="System.Buffers" Version="4.6.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\Server\Components\AICore\AICore.csproj" />
|
||||
<ProjectReference Include="..\..\..\Server\Components\ServerUtils\ServerUtils.csproj" />
|
||||
<ProjectReference Include="..\..\..\Shared\Shared.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!--
|
||||
DevExpress (GAC-installiert, v23.2) wird erst mit der PivotGrid-Matrix aktiviert:
|
||||
<ItemGroup>
|
||||
<Reference Include="DevExpress.Xpf.Core.v23.2" />
|
||||
<Reference Include="DevExpress.Xpf.Grid.v23.2" />
|
||||
<Reference Include="DevExpress.Xpf.PivotGrid.v23.2" />
|
||||
</ItemGroup>
|
||||
-->
|
||||
|
||||
</Project>
|
||||
8
Tools/AiResponseValidator/AiResponseValidator/App.xaml
Normal file
8
Tools/AiResponseValidator/AiResponseValidator/App.xaml
Normal file
@@ -0,0 +1,8 @@
|
||||
<Application
|
||||
x:Class="AiResponseValidator.App"
|
||||
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
xmlns:local="clr-namespace:AiResponseValidator"
|
||||
StartupUri="MainWindow.xaml">
|
||||
<Application.Resources />
|
||||
</Application>
|
||||
11
Tools/AiResponseValidator/AiResponseValidator/App.xaml.cs
Normal file
11
Tools/AiResponseValidator/AiResponseValidator/App.xaml.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
using System.Windows;
|
||||
|
||||
namespace AiResponseValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Interaction logic for App.xaml
|
||||
/// </summary>
|
||||
public partial class App : Application
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Windows;
|
||||
|
||||
[assembly: ThemeInfo(
|
||||
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
|
||||
//(used if a resource is not found in the page,
|
||||
// or application resource dictionaries)
|
||||
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
|
||||
//(used if a resource is not found in the page,
|
||||
// app, or any theme specific resource dictionaries)
|
||||
)]
|
||||
@@ -0,0 +1,41 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Konsistenz: alles fix, nur die Wiederholung läuft (die 'x'-Achse aus dem
|
||||
/// Juni-Entwurf). Auswertung interessiert die Streuung über die Läufe.
|
||||
/// </summary>
|
||||
public sealed class ConsistencyAxis : IAxis
|
||||
{
|
||||
private readonly int _Runs;
|
||||
|
||||
/// <param name="runs">Anzahl Wiederholungen; 0 = <see cref="TestPlan.Repeat"/> verwenden.</param>
|
||||
public ConsistencyAxis(int runs = 0)
|
||||
{
|
||||
_Runs = runs;
|
||||
}
|
||||
|
||||
public string Name => "Konsistenz";
|
||||
|
||||
public IEnumerable<TestCase> Expand(TestPlan plan)
|
||||
{
|
||||
var runs = _Runs > 0 ? _Runs : System.Math.Max(1, plan.Repeat);
|
||||
|
||||
foreach (var model in plan.Models)
|
||||
foreach (var request in plan.Requests)
|
||||
for (var run = 0; run < runs; run++)
|
||||
{
|
||||
yield return new TestCase(
|
||||
RequestId: request.Id,
|
||||
ModelId: model,
|
||||
InputVariant: "base",
|
||||
RunIndex: run,
|
||||
SystemPrompt: plan.SystemPrompt,
|
||||
UserMessage: request.UserMessage,
|
||||
Rule: request.Rule,
|
||||
Options: plan.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Eine Achse expandiert einen <see cref="TestPlan"/> zum kartesischen Produkt
|
||||
/// konkreter <see cref="TestCase"/>s. Der Runner kennt nur die flache Fallliste
|
||||
/// und weiß nichts von "Robustheit" vs. "Vergleich" – das ist genau der eine,
|
||||
/// austauschbare Regler.
|
||||
/// </summary>
|
||||
public interface IAxis
|
||||
{
|
||||
string Name { get; }
|
||||
|
||||
IEnumerable<TestCase> Expand(TestPlan plan);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Eine benannte Eingabevariante für den semantischen Robustheitsmodus –
|
||||
/// z.B. "widerspruch_1" mit einer bewusst regelverletzenden Nutzeraufforderung.
|
||||
/// </summary>
|
||||
public sealed record InputVariant(string Label, string UserMessage);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Modellvergleich: Anfragen-Satz fix, das Modell wird durchgetauscht.
|
||||
/// Auswertung interessiert Aggregat-Kennzahlen pro Modell (Erfolgsquote, Latenz).
|
||||
/// </summary>
|
||||
public sealed class ModelComparisonAxis : IAxis
|
||||
{
|
||||
public string Name => "Modellvergleich";
|
||||
|
||||
public IEnumerable<TestCase> Expand(TestPlan plan)
|
||||
{
|
||||
foreach (var model in plan.Models)
|
||||
foreach (var request in plan.Requests)
|
||||
for (var run = 0; run < System.Math.Max(1, plan.Repeat); run++)
|
||||
{
|
||||
yield return new TestCase(
|
||||
RequestId: request.Id,
|
||||
ModelId: model,
|
||||
InputVariant: "base",
|
||||
RunIndex: run,
|
||||
SystemPrompt: plan.SystemPrompt,
|
||||
UserMessage: request.UserMessage,
|
||||
Rule: request.Rule,
|
||||
Options: plan.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Zwei verschiedene Robustheitsbegriffe auf derselben Achse:
|
||||
/// </summary>
|
||||
public enum PressureMode
|
||||
{
|
||||
/// <summary>
|
||||
/// Reiner Kontextfenster-Druck: dieselbe Aufgabe mit zunehmend längerem
|
||||
/// Füll-/Rauschtext davor (wie <c>buildChunk</c> im AiTokenAnalyzer).
|
||||
/// </summary>
|
||||
ContextFill,
|
||||
|
||||
/// <summary>
|
||||
/// Semantischer Druck: zunehmend komplexere/widersprüchliche Eingaben,
|
||||
/// als explizit vorgegebene Varianten.
|
||||
/// </summary>
|
||||
SemanticPressure
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace AiResponseValidator.Core.Axes
|
||||
{
|
||||
/// <summary>
|
||||
/// Robustheit: Modell und Regel fix, die Eingabe wächst schrittweise. Interessant
|
||||
/// ist der Bruchpunkt – ab welcher Eingabegröße/-komplexität die Regel kippt.
|
||||
///
|
||||
/// Der <see cref="PressureMode"/> hält die zwei Robustheitsbegriffe sauber
|
||||
/// auseinander: <see cref="PressureMode.ContextFill"/> (Zeichenzahl-Stufen via
|
||||
/// Fülltext) bzw. <see cref="PressureMode.SemanticPressure"/> (vorgegebene Varianten).
|
||||
/// </summary>
|
||||
public sealed class RobustnessAxis : IAxis
|
||||
{
|
||||
private readonly PressureMode _Mode;
|
||||
private readonly IReadOnlyList<int> _Sizes;
|
||||
private readonly string _Filler;
|
||||
private readonly IReadOnlyList<InputVariant> _Variants;
|
||||
|
||||
/// <summary>Kontextfenster-Druck: dieselbe Anfrage, davor Fülltext auf N Zeichen.</summary>
|
||||
public static RobustnessAxis ContextFill(IReadOnlyList<int> sizes, string filler)
|
||||
=> new RobustnessAxis(PressureMode.ContextFill, sizes, filler ?? "", null);
|
||||
|
||||
/// <summary>Semantischer Druck: explizit vorgegebene, zunehmend härtere Eingaben.</summary>
|
||||
public static RobustnessAxis Semantic(IReadOnlyList<InputVariant> variants)
|
||||
=> new RobustnessAxis(PressureMode.SemanticPressure, null, null, variants);
|
||||
|
||||
private RobustnessAxis(PressureMode mode, IReadOnlyList<int> sizes, string filler, IReadOnlyList<InputVariant> variants)
|
||||
{
|
||||
_Mode = mode;
|
||||
_Sizes = sizes ?? new int[0];
|
||||
_Filler = filler;
|
||||
_Variants = variants ?? new InputVariant[0];
|
||||
}
|
||||
|
||||
public string Name => _Mode == PressureMode.ContextFill ? "Robustheit (Kontext)" : "Robustheit (Semantik)";
|
||||
|
||||
public IEnumerable<TestCase> Expand(TestPlan plan)
|
||||
{
|
||||
var runs = Math.Max(1, plan.Repeat);
|
||||
|
||||
foreach (var model in plan.Models)
|
||||
foreach (var request in plan.Requests)
|
||||
foreach (var variant in BuildVariants(request))
|
||||
for (var run = 0; run < runs; run++)
|
||||
{
|
||||
yield return new TestCase(
|
||||
RequestId: request.Id,
|
||||
ModelId: model,
|
||||
InputVariant: variant.Label,
|
||||
RunIndex: run,
|
||||
SystemPrompt: plan.SystemPrompt,
|
||||
UserMessage: variant.UserMessage,
|
||||
Rule: request.Rule,
|
||||
Options: plan.Options);
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<InputVariant> BuildVariants(RequestSpec request)
|
||||
{
|
||||
if (_Mode == PressureMode.SemanticPressure)
|
||||
return _Variants;
|
||||
|
||||
var list = new List<InputVariant>();
|
||||
|
||||
foreach (var size in _Sizes)
|
||||
{
|
||||
var padded = Pad(_Filler, size) + request.UserMessage;
|
||||
list.Add(new InputVariant($"ctx:{size}", padded));
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>Wiederholt/kürzt den Fülltext auf genau <paramref name="targetChars"/> Zeichen.</summary>
|
||||
private static string Pad(string filler, int targetChars)
|
||||
{
|
||||
if (targetChars <= 0 || string.IsNullOrEmpty(filler))
|
||||
return "";
|
||||
|
||||
var sb = new StringBuilder(targetChars + filler.Length);
|
||||
|
||||
while (sb.Length < targetChars)
|
||||
sb.Append(filler);
|
||||
|
||||
return sb.ToString(0, targetChars);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Ermöglicht C#-9-'record'/'init'-Accessoren unter net48.
|
||||
// Der Compiler erwartet diesen Typ; im .NET Framework ist er nicht vorhanden.
|
||||
namespace System.Runtime.CompilerServices
|
||||
{
|
||||
internal static class IsExternalInit { }
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using AiResponseValidator.Core.Rules;
|
||||
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Eine Anfrage aus dem "Anfragen-Satz" (linke Spalte des Juni-Entwurfs):
|
||||
/// eine User-Nachricht plus die Regel, die ihre Antwort erfüllen muss.
|
||||
/// </summary>
|
||||
public sealed record RequestSpec(string Id, string UserMessage, IRule Rule);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Ergebnis einer Prüfregel auf eine LLM-Antwort: bestanden oder nicht,
|
||||
/// samt Regelname und (bei Fehlschlag) einer kurzen Begründung.
|
||||
/// </summary>
|
||||
public sealed record RuleResult(bool Pass, string RuleName, string Detail)
|
||||
{
|
||||
public static RuleResult Ok(string ruleName)
|
||||
=> new RuleResult(true, ruleName, null);
|
||||
|
||||
public static RuleResult Fail(string ruleName, string detail)
|
||||
=> new RuleResult(false, ruleName, detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>
|
||||
/// Canary-Token-Regel für die Truncation-Erkennung: Ein eindeutiges Token wird
|
||||
/// am Ende von SystemPrompt oder Eingabe platziert und per Anweisung zurückverlangt.
|
||||
/// Fehlt es in der Antwort, wurde der Kontext vermutlich still abgeschnitten
|
||||
/// (passt zum <c>prompt_eval_count</c>-Bruch aus dem AiTokenAnalyzer).
|
||||
/// </summary>
|
||||
public sealed class CanaryRule : IRule
|
||||
{
|
||||
private readonly string _Token;
|
||||
|
||||
public CanaryRule(string token)
|
||||
{
|
||||
_Token = token ?? "";
|
||||
}
|
||||
|
||||
public string Name => "Canary";
|
||||
|
||||
public RuleResult Check(string response)
|
||||
{
|
||||
return (response ?? "").Contains(_Token)
|
||||
? RuleResult.Ok(Name)
|
||||
: RuleResult.Fail(Name, $"Canary-Token '{_Token}' fehlt (mögliche Truncation)");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using System;
|
||||
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>Antwort muss (oder darf nicht) einen bestimmten Teilstring enthalten.</summary>
|
||||
public sealed class ContainsRule : IRule
|
||||
{
|
||||
private readonly string _Needle;
|
||||
private readonly bool _MustContain;
|
||||
private readonly bool _IgnoreCase;
|
||||
|
||||
public ContainsRule(string needle, bool mustContain = true, bool ignoreCase = true)
|
||||
{
|
||||
_Needle = needle ?? "";
|
||||
_MustContain = mustContain;
|
||||
_IgnoreCase = ignoreCase;
|
||||
}
|
||||
|
||||
public string Name => _MustContain ? "Contains" : "NotContains";
|
||||
|
||||
public RuleResult Check(string response)
|
||||
{
|
||||
var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
var contains = (response ?? "").IndexOf(_Needle, comparison) >= 0;
|
||||
|
||||
if (contains == _MustContain)
|
||||
return RuleResult.Ok(Name);
|
||||
|
||||
return RuleResult.Fail(Name, _MustContain
|
||||
? $"'{_Needle}' fehlt in der Antwort"
|
||||
: $"'{_Needle}' unerwartet in der Antwort enthalten");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System;
|
||||
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>
|
||||
/// Ad-hoc-Regel aus einem Prädikat – nützlich für Sonderfälle, ohne eine eigene
|
||||
/// Regelklasse anlegen zu müssen.
|
||||
/// </summary>
|
||||
public sealed class DelegateRule : IRule
|
||||
{
|
||||
private readonly Func<string, bool> _Predicate;
|
||||
private readonly string _FailDetail;
|
||||
|
||||
public DelegateRule(string name, Func<string, bool> predicate, string failDetail = null)
|
||||
{
|
||||
Name = name ?? "Delegate";
|
||||
_Predicate = predicate ?? throw new ArgumentNullException(nameof(predicate));
|
||||
_FailDetail = failDetail ?? "Prädikat nicht erfüllt";
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
public RuleResult Check(string response)
|
||||
=> _Predicate(response ?? "") ? RuleResult.Ok(Name) : RuleResult.Fail(Name, _FailDetail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using System;
|
||||
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>Antwort muss exakt dem erwarteten Text entsprechen (getrimmt).</summary>
|
||||
public sealed class ExactRule : IRule
|
||||
{
|
||||
private readonly string _Expected;
|
||||
private readonly bool _IgnoreCase;
|
||||
|
||||
public ExactRule(string expected, bool ignoreCase = false)
|
||||
{
|
||||
_Expected = expected ?? "";
|
||||
_IgnoreCase = ignoreCase;
|
||||
}
|
||||
|
||||
public string Name => "Exact";
|
||||
|
||||
public RuleResult Check(string response)
|
||||
{
|
||||
var actual = (response ?? "").Trim();
|
||||
var comparison = _IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
|
||||
return string.Equals(actual, _Expected.Trim(), comparison)
|
||||
? RuleResult.Ok(Name)
|
||||
: RuleResult.Fail(Name, $"Erwartet '{_Expected}', erhalten '{Shorten(actual)}'");
|
||||
}
|
||||
|
||||
internal static string Shorten(string s, int max = 60)
|
||||
=> s == null ? "" : (s.Length <= max ? s : s.Substring(0, max) + "…");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>
|
||||
/// Eine austauschbare Prüfregel auf den Text einer LLM-Antwort.
|
||||
/// Konkrete Regeln: <see cref="ExactRule"/>, <see cref="ContainsRule"/>,
|
||||
/// <see cref="RegexRule"/>, <see cref="CanaryRule"/>, <see cref="JsonSchemaRule"/>,
|
||||
/// <see cref="DelegateRule"/>.
|
||||
/// </summary>
|
||||
public interface IRule
|
||||
{
|
||||
/// <summary>Kurzer, in der Ergebnis-Matrix anzeigbarer Name der Regel.</summary>
|
||||
string Name { get; }
|
||||
|
||||
/// <summary>Prüft die Antwort und liefert bestanden/nicht bestanden zurück.</summary>
|
||||
RuleResult Check(string response);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>
|
||||
/// Leichtgewichtige JSON-Prüfung ohne externe Abhängigkeit: prüft strukturelle
|
||||
/// Wohlgeformtheit (ausbalancierte Klammern unter Beachtung von Strings/Escapes)
|
||||
/// und optional das Vorhandensein erwarteter Top-Level-Schlüssel.
|
||||
///
|
||||
/// Bewusst noch keine vollständige JSON-Schema-Validierung – das ist der
|
||||
/// vorgesehene Erweiterungspunkt (z.B. später via Newtonsoft.Json.Schema).
|
||||
/// </summary>
|
||||
public sealed class JsonSchemaRule : IRule
|
||||
{
|
||||
private readonly IReadOnlyList<string> _RequiredProperties;
|
||||
|
||||
public JsonSchemaRule(params string[] requiredProperties)
|
||||
{
|
||||
_RequiredProperties = requiredProperties ?? new string[0];
|
||||
}
|
||||
|
||||
public string Name => "Json";
|
||||
|
||||
public RuleResult Check(string response)
|
||||
{
|
||||
var text = (response ?? "").Trim();
|
||||
|
||||
if (text.Length == 0)
|
||||
return RuleResult.Fail(Name, "Leere Antwort");
|
||||
|
||||
if (!IsWellFormed(text))
|
||||
return RuleResult.Fail(Name, "Kein wohlgeformtes JSON (unausgeglichene Klammern)");
|
||||
|
||||
var missing = _RequiredProperties
|
||||
.Where(p => !text.Contains("\"" + p + "\""))
|
||||
.ToList();
|
||||
|
||||
return missing.Count == 0
|
||||
? RuleResult.Ok(Name)
|
||||
: RuleResult.Fail(Name, "Fehlende Schlüssel: " + string.Join(", ", missing));
|
||||
}
|
||||
|
||||
/// <summary>Klammerbalance über { } und [ ], Strings und Escapes werden übersprungen.</summary>
|
||||
private static bool IsWellFormed(string s)
|
||||
{
|
||||
var stack = new Stack<char>();
|
||||
var in_string = false;
|
||||
var escaped = false;
|
||||
|
||||
foreach (var c in s)
|
||||
{
|
||||
if (in_string)
|
||||
{
|
||||
if (escaped) escaped = false;
|
||||
else if (c == '\\') escaped = true;
|
||||
else if (c == '"') in_string = false;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (c)
|
||||
{
|
||||
case '"': in_string = true; break;
|
||||
case '{': stack.Push('}'); break;
|
||||
case '[': stack.Push(']'); break;
|
||||
case '}':
|
||||
case ']':
|
||||
if (stack.Count == 0 || stack.Pop() != c)
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return !in_string && stack.Count == 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace AiResponseValidator.Core.Rules
|
||||
{
|
||||
/// <summary>Antwort muss (oder darf nicht) auf ein Regex-Muster passen.</summary>
|
||||
public sealed class RegexRule : IRule
|
||||
{
|
||||
private readonly Regex _Regex;
|
||||
private readonly bool _MustMatch;
|
||||
private readonly string _Pattern;
|
||||
|
||||
public RegexRule(string pattern, bool mustMatch = true, RegexOptions options = RegexOptions.Singleline)
|
||||
{
|
||||
_Pattern = pattern ?? "";
|
||||
_Regex = new Regex(_Pattern, options);
|
||||
_MustMatch = mustMatch;
|
||||
}
|
||||
|
||||
public string Name => "Regex";
|
||||
|
||||
public RuleResult Check(string response)
|
||||
{
|
||||
var matches = _Regex.IsMatch(response ?? "");
|
||||
|
||||
if (matches == _MustMatch)
|
||||
return RuleResult.Ok(Name);
|
||||
|
||||
return RuleResult.Fail(Name, _MustMatch
|
||||
? $"Muster '{_Pattern}' nicht gefunden"
|
||||
: $"Muster '{_Pattern}' unerwartet gefunden");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Messwerte eines einzelnen Laufs. <see cref="PromptEvalCount"/> ist die vom
|
||||
/// Backend gemeldete Anzahl tatsächlich verarbeiteter Prompt-Tokens und dient
|
||||
/// (wie im AiTokenAnalyzer) zur Erkennung stiller Truncation.
|
||||
/// </summary>
|
||||
public sealed record RunMetrics(
|
||||
int InputChars,
|
||||
long? PromptEvalCount,
|
||||
long? EvalCount,
|
||||
int? DurationMs)
|
||||
{
|
||||
public static readonly RunMetrics Empty = new RunMetrics(0, null, null, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Backend-Optionen für einen Lauf. <see cref="NumCtx"/> steuert die
|
||||
/// Kontextfenstergröße (Ollama <c>options.num_ctx</c>).
|
||||
/// </summary>
|
||||
public sealed record RunOptions(int? NumCtx = null, string CustomerId = null)
|
||||
{
|
||||
public static readonly RunOptions Default = new RunOptions();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AiResponseValidator.Core.Runner
|
||||
{
|
||||
/// <summary>
|
||||
/// Führt genau einen <see cref="TestCase"/> aus und wendet dessen Regel an.
|
||||
/// Der geteilte Kern: run(request, model, options) → response → prüfregel → pass/fail.
|
||||
/// </summary>
|
||||
public interface ITestRunner
|
||||
{
|
||||
Task<TestRun> RunAsync(TestCase testCase, CancellationToken ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AICore.Facade.LLM;
|
||||
|
||||
namespace AiResponseValidator.Core.Runner
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="ITestRunner"/> gegen den Ollama-Endpunkt (wiederverwendet den
|
||||
/// <see cref="OllamaApiClient"/> aus AICore, wie der AiTokenAnalyzer).
|
||||
/// </summary>
|
||||
public sealed class OllamaTestRunner : ITestRunner
|
||||
{
|
||||
private readonly string _Url;
|
||||
private readonly string _ApiKey;
|
||||
|
||||
public OllamaTestRunner(string url, string apiKey)
|
||||
{
|
||||
_Url = (url ?? "").Trim();
|
||||
_ApiKey = (apiKey ?? "").Trim();
|
||||
}
|
||||
|
||||
public async Task<TestRun> RunAsync(TestCase testCase, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var payload = BuildPayload(testCase);
|
||||
var input_chars = testCase.UserMessage?.Length ?? 0;
|
||||
|
||||
try
|
||||
{
|
||||
var result = await Task.Run(() =>
|
||||
{
|
||||
var client = new OllamaApiClient(_Url, _ApiKey);
|
||||
var response = client.SendAiMessageRequest(
|
||||
payload,
|
||||
out string message,
|
||||
out string _,
|
||||
out int? duration,
|
||||
out long? prompt_eval_count,
|
||||
out long? eval_count);
|
||||
|
||||
var ok = response != null && response.Success;
|
||||
var error = ok ? null : (response != null ? response.ToErrorString() : "Keine Antwort vom Backend");
|
||||
|
||||
return new RawResult(message, duration, prompt_eval_count, eval_count, error);
|
||||
}, ct);
|
||||
|
||||
var metrics = new RunMetrics(input_chars, result.PromptEvalCount, result.EvalCount, result.DurationMs);
|
||||
|
||||
if (result.Error != null)
|
||||
return Failed(testCase, metrics, result.Error);
|
||||
|
||||
var rule = testCase.Rule?.Check(result.Message ?? "");
|
||||
|
||||
return new TestRun(
|
||||
testCase.RequestId, testCase.ModelId, testCase.InputVariant, testCase.RunIndex,
|
||||
result.Message, metrics, rule);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Failed(testCase, new RunMetrics(input_chars, null, null, null), ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static TestRun Failed(TestCase c, RunMetrics metrics, string error)
|
||||
=> new TestRun(c.RequestId, c.ModelId, c.InputVariant, c.RunIndex, null, metrics, null, error);
|
||||
|
||||
private static object BuildPayload(TestCase c)
|
||||
{
|
||||
var messages = new List<object>();
|
||||
|
||||
if (!string.IsNullOrEmpty(c.SystemPrompt))
|
||||
messages.Add(new { role = "system", content = c.SystemPrompt });
|
||||
|
||||
messages.Add(new { role = "user", content = c.UserMessage });
|
||||
|
||||
var payload = new Dictionary<string, object>
|
||||
{
|
||||
["model"] = c.ModelId,
|
||||
["messages"] = messages,
|
||||
["requestid"] = Guid.NewGuid().ToString("N")
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(c.Options?.CustomerId))
|
||||
payload["customerid"] = c.Options.CustomerId;
|
||||
|
||||
if (c.Options?.NumCtx is int ctx)
|
||||
payload["options"] = new { num_ctx = ctx };
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
private readonly struct RawResult
|
||||
{
|
||||
public RawResult(string message, int? durationMs, long? promptEvalCount, long? evalCount, string error)
|
||||
{
|
||||
Message = message;
|
||||
DurationMs = durationMs;
|
||||
PromptEvalCount = promptEvalCount;
|
||||
EvalCount = evalCount;
|
||||
Error = error;
|
||||
}
|
||||
|
||||
public string Message { get; }
|
||||
public int? DurationMs { get; }
|
||||
public long? PromptEvalCount { get; }
|
||||
public long? EvalCount { get; }
|
||||
public string Error { get; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AiResponseValidator.Core.Axes;
|
||||
|
||||
namespace AiResponseValidator.Core.Runner
|
||||
{
|
||||
/// <summary>
|
||||
/// Verbindet Achse, Plan und Runner: expandiert den Plan entlang der gewählten
|
||||
/// Achse zu Fällen und führt sie (sequenziell) aus. Jeder fertige
|
||||
/// <see cref="TestRun"/> wird sofort über <paramref name="onCompleted"/> gemeldet,
|
||||
/// damit die UI live grün/rot färben kann – der Runner selbst bleibt UI-frei.
|
||||
/// </summary>
|
||||
public sealed class TestExecutor
|
||||
{
|
||||
private readonly ITestRunner _Runner;
|
||||
|
||||
public TestExecutor(ITestRunner runner)
|
||||
{
|
||||
_Runner = runner;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TestRun>> RunAsync(
|
||||
TestPlan plan,
|
||||
IAxis axis,
|
||||
IProgress<TestRun> onCompleted,
|
||||
IProgress<ExecutionProgress> onProgress,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var cases = axis.Expand(plan).ToList();
|
||||
var results = new List<TestRun>(cases.Count);
|
||||
|
||||
for (var i = 0; i < cases.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
onProgress?.Report(new ExecutionProgress(i, cases.Count, cases[i]));
|
||||
|
||||
var run = await _Runner.RunAsync(cases[i], ct);
|
||||
|
||||
results.Add(run);
|
||||
onCompleted?.Report(run);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fortschrittsmeldung: Fall <see cref="Index"/> von <see cref="Total"/> startet.</summary>
|
||||
public sealed record ExecutionProgress(int Index, int Total, TestCase Current);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using AiResponseValidator.Core.Rules;
|
||||
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Eine einzelne, ausführbare Anweisung an den Runner – das Ergebnis der
|
||||
/// Expansion eines <see cref="TestPlan"/> entlang einer <see cref="Axes.IAxis"/>.
|
||||
/// Die vier Identitätsfelder (RequestId, ModelId, InputVariant, RunIndex) sind
|
||||
/// exakt die Achsen der Ergebnis-Matrix.
|
||||
/// </summary>
|
||||
public sealed record TestCase(
|
||||
string RequestId,
|
||||
string ModelId,
|
||||
string InputVariant,
|
||||
int RunIndex,
|
||||
string SystemPrompt,
|
||||
string UserMessage,
|
||||
IRule Rule,
|
||||
RunOptions Options);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Die gemeinsame Konfiguration aller Szenarien. Eine <see cref="Axes.IAxis"/>
|
||||
/// entscheidet, welche dieser Dimensionen durchlaufen wird – der Plan selbst
|
||||
/// ist szenario-neutral.
|
||||
/// </summary>
|
||||
public sealed class TestPlan
|
||||
{
|
||||
public string SystemPrompt { get; set; }
|
||||
|
||||
/// <summary>Der Anfragen-Satz (linke Spalte im Entwurf).</summary>
|
||||
public IReadOnlyList<RequestSpec> Requests { get; set; } = new List<RequestSpec>();
|
||||
|
||||
/// <summary>Die zu prüfenden Modelle.</summary>
|
||||
public IReadOnlyList<string> Models { get; set; } = new List<string>();
|
||||
|
||||
public RunOptions Options { get; set; } = RunOptions.Default;
|
||||
|
||||
/// <summary>Wiederholungen pro Fall (die 'x'-Achse aus dem Juni-Entwurf).</summary>
|
||||
public int Repeat { get; set; } = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
namespace AiResponseValidator.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Das Ergebnis genau eines ausgeführten <see cref="TestCase"/> – die eine, flache
|
||||
/// Zeile, aus der jede Szenario-Sicht nur eine andere GROUP-BY-Projektion ist.
|
||||
/// Bewusst unveränderlich, damit sie sich gefahrlos als Roh-Datensatz exportieren
|
||||
/// und später erneut auswerten lässt (ohne die LLMs neu zu befragen).
|
||||
/// </summary>
|
||||
public sealed record TestRun(
|
||||
string RequestId,
|
||||
string ModelId,
|
||||
string InputVariant,
|
||||
int RunIndex,
|
||||
string Response,
|
||||
RunMetrics Metrics,
|
||||
RuleResult Rule,
|
||||
string Error = null)
|
||||
{
|
||||
/// <summary>Der Request lief technisch durch (kein Transport-/Backend-Fehler).</summary>
|
||||
public bool Completed => Error == null;
|
||||
|
||||
/// <summary>Grün in der Matrix: durchgelaufen UND Regel erfüllt.</summary>
|
||||
public bool Passed => Completed && Rule != null && Rule.Pass;
|
||||
}
|
||||
}
|
||||
119
Tools/AiResponseValidator/AiResponseValidator/MainWindow.xaml
Normal file
119
Tools/AiResponseValidator/AiResponseValidator/MainWindow.xaml
Normal file
@@ -0,0 +1,119 @@
|
||||
<Window
|
||||
x:Class="AiResponseValidator.MainWindow"
|
||||
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
Title="AI Response Validator" Height="720" Width="1180">
|
||||
|
||||
<DockPanel>
|
||||
|
||||
<!-- Toolbar -->
|
||||
<ToolBarTray DockPanel.Dock="Top">
|
||||
<ToolBar>
|
||||
<TextBlock Text="URL:" VerticalAlignment="Center" Margin="4,0" />
|
||||
<TextBox Text="{Binding Url, UpdateSourceTrigger=PropertyChanged}" Width="200" />
|
||||
<TextBlock Text="API-Key:" VerticalAlignment="Center" Margin="8,0,4,0" />
|
||||
<TextBox Text="{Binding ApiKey, UpdateSourceTrigger=PropertyChanged}" Width="120" />
|
||||
<Separator />
|
||||
<TextBlock Text="Szenario:" VerticalAlignment="Center" Margin="4,0" />
|
||||
<ComboBox ItemsSource="{Binding Scenarios}" SelectedItem="{Binding Scenario}" Width="150" />
|
||||
<Separator />
|
||||
<Button Content="▶ Ausführen" Command="{Binding RunCommand}" Padding="8,2" />
|
||||
<Button Content="■ Abbrechen" Command="{Binding CancelCommand}" Padding="8,2" Margin="4,0,0,0" />
|
||||
</ToolBar>
|
||||
</ToolBarTray>
|
||||
|
||||
<!-- Status -->
|
||||
<StatusBar DockPanel.Dock="Bottom">
|
||||
<StatusBarItem>
|
||||
<TextBlock Text="{Binding StatusText}" />
|
||||
</StatusBarItem>
|
||||
</StatusBar>
|
||||
|
||||
<Grid>
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="360" />
|
||||
<ColumnDefinition Width="4" />
|
||||
<ColumnDefinition Width="*" />
|
||||
</Grid.ColumnDefinitions>
|
||||
|
||||
<!-- Links: Konfiguration / Anfragen-Satz (Grundgerüst: eine Anfrage) -->
|
||||
<ScrollViewer Grid.Column="0" VerticalScrollBarVisibility="Auto">
|
||||
<StackPanel Margin="8">
|
||||
<TextBlock Text="System-Prompt" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding SystemPrompt, UpdateSourceTrigger=PropertyChanged}"
|
||||
AcceptsReturn="True" TextWrapping="Wrap" Height="120"
|
||||
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
|
||||
|
||||
<TextBlock Text="Modelle (Komma/Zeile getrennt)" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding ModelsText, UpdateSourceTrigger=PropertyChanged}"
|
||||
AcceptsReturn="True" TextWrapping="Wrap" Height="60"
|
||||
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
|
||||
|
||||
<TextBlock Text="Anfrage (User-Nachricht)" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding RequestMessage, UpdateSourceTrigger=PropertyChanged}"
|
||||
AcceptsReturn="True" TextWrapping="Wrap" Height="100"
|
||||
VerticalScrollBarVisibility="Auto" Margin="0,2,0,8" />
|
||||
|
||||
<TextBlock Text="Regel" FontWeight="Bold" />
|
||||
<ComboBox ItemsSource="{Binding RuleKinds}" SelectedItem="{Binding Rule}" Margin="0,2,0,4" />
|
||||
<TextBox Text="{Binding RuleArgument, UpdateSourceTrigger=PropertyChanged}"
|
||||
TextWrapping="Wrap" Margin="0,0,0,8"
|
||||
ToolTip="Contains/Exact/Regex: Text/Muster · Canary: Token · Json: erwartete Schlüssel" />
|
||||
|
||||
<TextBlock Text="Größen (nur Robustheit/Kontext)" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding SizesText, UpdateSourceTrigger=PropertyChanged}" Margin="0,2,0,8" />
|
||||
|
||||
<Grid Margin="0,0,0,8">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="*" />
|
||||
<ColumnDefinition Width="*" />
|
||||
</Grid.ColumnDefinitions>
|
||||
<StackPanel Grid.Column="0" Margin="0,0,4,0">
|
||||
<TextBlock Text="Wiederholungen" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding Repeat, UpdateSourceTrigger=PropertyChanged}" />
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="1" Margin="4,0,0,0">
|
||||
<TextBlock Text="num_ctx (optional)" FontWeight="Bold" />
|
||||
<TextBox Text="{Binding NumCtxText, UpdateSourceTrigger=PropertyChanged}" />
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<GridSplitter Grid.Column="1" Width="4" HorizontalAlignment="Stretch" Background="#DDD" />
|
||||
|
||||
<!-- Mitte: Ergebnis-Matrix (Grundgerüst: flache Liste; PivotGrid folgt) -->
|
||||
<DataGrid Grid.Column="2" ItemsSource="{Binding Results}" AutoGenerateColumns="False"
|
||||
IsReadOnly="True" CanUserAddRows="False" Margin="4">
|
||||
<DataGrid.RowStyle>
|
||||
<Style TargetType="DataGridRow">
|
||||
<Style.Triggers>
|
||||
<DataTrigger Binding="{Binding Passed}" Value="True">
|
||||
<Setter Property="Background" Value="#D6F5D6" />
|
||||
</DataTrigger>
|
||||
<DataTrigger Binding="{Binding Passed}" Value="False">
|
||||
<Setter Property="Background" Value="#F8D6D6" />
|
||||
</DataTrigger>
|
||||
<DataTrigger Binding="{Binding Completed}" Value="False">
|
||||
<Setter Property="Background" Value="#FBE6C2" />
|
||||
</DataTrigger>
|
||||
</Style.Triggers>
|
||||
</Style>
|
||||
</DataGrid.RowStyle>
|
||||
<DataGrid.Columns>
|
||||
<DataGridTextColumn Header="Modell" Binding="{Binding ModelId}" Width="160" />
|
||||
<DataGridTextColumn Header="Variante" Binding="{Binding InputVariant}" Width="90" />
|
||||
<DataGridTextColumn Header="Lauf" Binding="{Binding RunIndex}" Width="45" />
|
||||
<DataGridTextColumn Header="Regel" Binding="{Binding Rule.Name}" Width="80" />
|
||||
<DataGridTextColumn Header="Pass" Binding="{Binding Passed}" Width="50" />
|
||||
<DataGridTextColumn Header="Detail" Binding="{Binding Rule.Detail}" Width="180" />
|
||||
<DataGridTextColumn Header="prompt_eval" Binding="{Binding Metrics.PromptEvalCount}" Width="80" />
|
||||
<DataGridTextColumn Header="ms" Binding="{Binding Metrics.DurationMs}" Width="60" />
|
||||
<DataGridTextColumn Header="Antwort" Binding="{Binding Response}" Width="*" />
|
||||
<DataGridTextColumn Header="Fehler" Binding="{Binding Error}" Width="160" />
|
||||
</DataGrid.Columns>
|
||||
</DataGrid>
|
||||
|
||||
</Grid>
|
||||
</DockPanel>
|
||||
</Window>
|
||||
@@ -0,0 +1,30 @@
|
||||
using System.ComponentModel;
|
||||
using System.Windows;
|
||||
using AiResponseValidator.ViewModels;
|
||||
|
||||
namespace AiResponseValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Interaction logic for MainWindow.xaml. Setzt den <see cref="MainViewModel"/>
|
||||
/// als DataContext und lädt/speichert dessen Eingaben beim Start/Schließen.
|
||||
/// </summary>
|
||||
public partial class MainWindow : Window
|
||||
{
|
||||
private readonly MainViewModel _ViewModel;
|
||||
|
||||
public MainWindow()
|
||||
{
|
||||
InitializeComponent();
|
||||
|
||||
_ViewModel = new MainViewModel();
|
||||
_ViewModel.LoadSettings();
|
||||
DataContext = _ViewModel;
|
||||
}
|
||||
|
||||
protected override void OnClosing(CancelEventArgs e)
|
||||
{
|
||||
_ViewModel.SaveSettings();
|
||||
base.OnClosing(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using AiResponseValidator.ViewModels;
|
||||
|
||||
namespace AiResponseValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Beim Schließen gespeicherte / beim Öffnen geladene Eingabewerte
|
||||
/// (wie im AiTokenAnalyzer, via XmlSerializer in %AppData%).
|
||||
/// </summary>
|
||||
public class UserSettings
|
||||
{
|
||||
public string Url { get; set; }
|
||||
public string ApiKey { get; set; }
|
||||
public string SystemPrompt { get; set; }
|
||||
public string ModelsText { get; set; }
|
||||
public string RequestMessage { get; set; }
|
||||
public RuleKind Rule { get; set; }
|
||||
public string RuleArgument { get; set; }
|
||||
public ScenarioKind Scenario { get; set; }
|
||||
public string SizesText { get; set; }
|
||||
public string FillerText { get; set; }
|
||||
public int Repeat { get; set; } = 1;
|
||||
public string NumCtxText { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using System.Windows.Input;
|
||||
|
||||
namespace AiResponseValidator.ViewModels
|
||||
{
|
||||
/// <summary>ICommand für asynchrone Aktionen; sperrt sich selbst während der Ausführung.</summary>
|
||||
public sealed class AsyncRelayCommand : ICommand
|
||||
{
|
||||
private readonly Func<Task> _ExecuteAsync;
|
||||
private readonly Func<bool> _CanExecute;
|
||||
private bool _IsRunning;
|
||||
|
||||
public AsyncRelayCommand(Func<Task> executeAsync, Func<bool> canExecute = null)
|
||||
{
|
||||
_ExecuteAsync = executeAsync ?? throw new ArgumentNullException(nameof(executeAsync));
|
||||
_CanExecute = canExecute;
|
||||
}
|
||||
|
||||
public bool CanExecute(object parameter) => !_IsRunning && (_CanExecute == null || _CanExecute());
|
||||
|
||||
public async void Execute(object parameter)
|
||||
{
|
||||
_IsRunning = true;
|
||||
CommandManager.InvalidateRequerySuggested();
|
||||
|
||||
try
|
||||
{
|
||||
await _ExecuteAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_IsRunning = false;
|
||||
CommandManager.InvalidateRequerySuggested();
|
||||
}
|
||||
}
|
||||
|
||||
public event EventHandler CanExecuteChanged
|
||||
{
|
||||
add { CommandManager.RequerySuggested += value; }
|
||||
remove { CommandManager.RequerySuggested -= value; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Serialization;
|
||||
using AiResponseValidator.Core;
|
||||
using AiResponseValidator.Core.Axes;
|
||||
using AiResponseValidator.Core.Rules;
|
||||
using AiResponseValidator.Core.Runner;
|
||||
|
||||
namespace AiResponseValidator.ViewModels
|
||||
{
|
||||
/// <summary>
|
||||
/// Bindeglied zwischen Toolbar-Konfiguration, Achsenwahl und der flachen
|
||||
/// Ergebnis-Liste. Hält KEINE Backend-Logik – die liegt komplett im Core.
|
||||
///
|
||||
/// Erstes Grundgerüst: ein einzelner Anfrage-Editor. Der volle Anfragen-Satz
|
||||
/// (mehrere <see cref="RequestSpec"/>) sowie die PivotGrid-Matrix folgen als
|
||||
/// nächster Schritt.
|
||||
/// </summary>
|
||||
public sealed class MainViewModel : ViewModelBase
|
||||
{
|
||||
private string _Url = "http://localhost:11434";
|
||||
private string _ApiKey = "";
|
||||
private string _SystemPrompt = "";
|
||||
private string _ModelsText = "gemma3_12b_max_context:latest";
|
||||
private string _RequestMessage = "";
|
||||
private RuleKind _Rule = RuleKind.Contains;
|
||||
private string _RuleArgument = "";
|
||||
private ScenarioKind _Scenario = ScenarioKind.Modellvergleich;
|
||||
private string _SizesText = "50k, 100k, 150k";
|
||||
private string _FillerText = "";
|
||||
private int _Repeat = 1;
|
||||
private string _NumCtxText = "";
|
||||
private string _StatusText = "Bereit.";
|
||||
private CancellationTokenSource _Cts;
|
||||
|
||||
public MainViewModel()
|
||||
{
|
||||
RunCommand = new AsyncRelayCommand(RunAsync, () => !IsRunning);
|
||||
CancelCommand = new RelayCommand(Cancel, () => IsRunning);
|
||||
|
||||
try
|
||||
{
|
||||
_FillerText = File.ReadAllText("derprozess.txt");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_FillerText = "Der Affe ist gross. ";
|
||||
}
|
||||
}
|
||||
|
||||
public ObservableCollection<TestRun> Results { get; } = new ObservableCollection<TestRun>();
|
||||
|
||||
public string Url { get => _Url; set => Set(ref _Url, value); }
|
||||
public string ApiKey { get => _ApiKey; set => Set(ref _ApiKey, value); }
|
||||
public string SystemPrompt { get => _SystemPrompt; set => Set(ref _SystemPrompt, value); }
|
||||
public string ModelsText { get => _ModelsText; set => Set(ref _ModelsText, value); }
|
||||
public string RequestMessage { get => _RequestMessage; set => Set(ref _RequestMessage, value); }
|
||||
public RuleKind Rule { get => _Rule; set => Set(ref _Rule, value); }
|
||||
public string RuleArgument { get => _RuleArgument; set => Set(ref _RuleArgument, value); }
|
||||
public ScenarioKind Scenario { get => _Scenario; set => Set(ref _Scenario, value); }
|
||||
public string SizesText { get => _SizesText; set => Set(ref _SizesText, value); }
|
||||
public string FillerText { get => _FillerText; set => Set(ref _FillerText, value); }
|
||||
public int Repeat { get => _Repeat; set => Set(ref _Repeat, value); }
|
||||
public string NumCtxText { get => _NumCtxText; set => Set(ref _NumCtxText, value); }
|
||||
|
||||
public string StatusText { get => _StatusText; set => Set(ref _StatusText, value); }
|
||||
|
||||
private bool _IsRunning;
|
||||
public bool IsRunning
|
||||
{
|
||||
get => _IsRunning;
|
||||
private set => Set(ref _IsRunning, value);
|
||||
}
|
||||
|
||||
public AsyncRelayCommand RunCommand { get; }
|
||||
public RelayCommand CancelCommand { get; }
|
||||
|
||||
public IEnumerable<ScenarioKind> Scenarios => Enum.GetValues(typeof(ScenarioKind)).Cast<ScenarioKind>();
|
||||
public IEnumerable<RuleKind> RuleKinds => Enum.GetValues(typeof(RuleKind)).Cast<RuleKind>();
|
||||
|
||||
private static string GetSettingsPath()
|
||||
=> Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "AiResponseValidator", "settings.xml");
|
||||
|
||||
/// <summary>Lädt die zuletzt gespeicherten Eingaben; defekte/fehlende Datei = Defaults.</summary>
|
||||
public void LoadSettings()
|
||||
{
|
||||
try
|
||||
{
|
||||
var path = GetSettingsPath();
|
||||
|
||||
if (!File.Exists(path))
|
||||
return;
|
||||
|
||||
UserSettings s;
|
||||
|
||||
using (var stream = File.OpenRead(path))
|
||||
s = (UserSettings)new XmlSerializer(typeof(UserSettings)).Deserialize(stream);
|
||||
|
||||
if (s == null)
|
||||
return;
|
||||
|
||||
Url = s.Url ?? Url;
|
||||
ApiKey = s.ApiKey ?? ApiKey;
|
||||
SystemPrompt = s.SystemPrompt ?? SystemPrompt;
|
||||
ModelsText = s.ModelsText ?? ModelsText;
|
||||
RequestMessage = s.RequestMessage ?? RequestMessage;
|
||||
Rule = s.Rule;
|
||||
RuleArgument = s.RuleArgument ?? RuleArgument;
|
||||
Scenario = s.Scenario;
|
||||
SizesText = s.SizesText ?? SizesText;
|
||||
FillerText = s.FillerText ?? FillerText;
|
||||
Repeat = s.Repeat > 0 ? s.Repeat : Repeat;
|
||||
NumCtxText = s.NumCtxText ?? NumCtxText;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Defekte Settings-Datei ignorieren, Defaults verwenden.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Speichert die aktuellen Eingaben; Fehler dürfen das Schließen nicht verhindern.</summary>
|
||||
public void SaveSettings()
|
||||
{
|
||||
try
|
||||
{
|
||||
var s = new UserSettings
|
||||
{
|
||||
Url = Url,
|
||||
ApiKey = ApiKey,
|
||||
SystemPrompt = SystemPrompt,
|
||||
ModelsText = ModelsText,
|
||||
RequestMessage = RequestMessage,
|
||||
Rule = Rule,
|
||||
RuleArgument = RuleArgument,
|
||||
Scenario = Scenario,
|
||||
SizesText = SizesText,
|
||||
FillerText = FillerText,
|
||||
Repeat = Repeat,
|
||||
NumCtxText = NumCtxText
|
||||
};
|
||||
|
||||
var path = GetSettingsPath();
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path));
|
||||
|
||||
using (var stream = File.Create(path))
|
||||
new XmlSerializer(typeof(UserSettings)).Serialize(stream, s);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Speichern der Settings darf das Schließen nicht verhindern.
|
||||
}
|
||||
}
|
||||
|
||||
private void Cancel()
|
||||
{
|
||||
_Cts?.Cancel();
|
||||
StatusText = "Abbruch angefordert...";
|
||||
}
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
var models = SplitList(ModelsText);
|
||||
|
||||
if (models.Count == 0)
|
||||
{
|
||||
StatusText = "Bitte mindestens ein Modell angeben.";
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(RequestMessage))
|
||||
{
|
||||
StatusText = "Bitte eine Anfrage (User-Nachricht) angeben.";
|
||||
return;
|
||||
}
|
||||
|
||||
var plan = BuildPlan(models);
|
||||
IAxis axis;
|
||||
|
||||
try
|
||||
{
|
||||
axis = BuildAxis();
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
StatusText = ex.Message;
|
||||
return;
|
||||
}
|
||||
|
||||
Results.Clear();
|
||||
IsRunning = true;
|
||||
_Cts = new CancellationTokenSource();
|
||||
|
||||
var executor = new TestExecutor(new OllamaTestRunner(Url, ApiKey));
|
||||
var onCompleted = new Progress<TestRun>(r => Results.Add(r));
|
||||
var onProgress = new Progress<ExecutionProgress>(p =>
|
||||
StatusText = $"[{axis.Name}] Fall {p.Index + 1}/{p.Total}: {p.Current.ModelId}, Variante '{p.Current.InputVariant}'...");
|
||||
|
||||
try
|
||||
{
|
||||
var all = await executor.RunAsync(plan, axis, onCompleted, onProgress, _Cts.Token);
|
||||
var passed = all.Count(r => r.Passed);
|
||||
StatusText = $"Fertig: {passed}/{all.Count} bestanden.";
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
StatusText = $"Abgebrochen ({Results.Count} Läufe erfasst).";
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
StatusText = "Fehler: " + ex.Message;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_Cts.Dispose();
|
||||
_Cts = null;
|
||||
IsRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
private TestPlan BuildPlan(IReadOnlyList<string> models)
|
||||
{
|
||||
int? num_ctx = null;
|
||||
|
||||
if (int.TryParse((NumCtxText ?? "").Trim(), out var ctx) && ctx > 0)
|
||||
num_ctx = ctx;
|
||||
|
||||
var request = new RequestSpec("req1", RequestMessage, BuildRule());
|
||||
|
||||
return new TestPlan
|
||||
{
|
||||
SystemPrompt = SystemPrompt,
|
||||
Requests = new[] { request },
|
||||
Models = models,
|
||||
Options = new RunOptions(num_ctx),
|
||||
Repeat = Math.Max(1, Repeat)
|
||||
};
|
||||
}
|
||||
|
||||
private IRule BuildRule()
|
||||
{
|
||||
var arg = RuleArgument ?? "";
|
||||
|
||||
switch (Rule)
|
||||
{
|
||||
case RuleKind.Contains: return new ContainsRule(arg, mustContain: true);
|
||||
case RuleKind.NotContains: return new ContainsRule(arg, mustContain: false);
|
||||
case RuleKind.Exact: return new ExactRule(arg);
|
||||
case RuleKind.Regex: return new RegexRule(arg);
|
||||
case RuleKind.Canary: return new CanaryRule(arg);
|
||||
case RuleKind.Json: return new JsonSchemaRule(SplitList(arg).ToArray());
|
||||
default: return new ContainsRule(arg);
|
||||
}
|
||||
}
|
||||
|
||||
private IAxis BuildAxis()
|
||||
{
|
||||
switch (Scenario)
|
||||
{
|
||||
case ScenarioKind.Modellvergleich:
|
||||
return new ModelComparisonAxis();
|
||||
|
||||
case ScenarioKind.Konsistenz:
|
||||
return new ConsistencyAxis();
|
||||
|
||||
case ScenarioKind.RobustheitKontext:
|
||||
return RobustnessAxis.ContextFill(ParseSizes(SizesText), FillerText);
|
||||
|
||||
case ScenarioKind.RobustheitSemantik:
|
||||
// Grundgerüst: eine Variante aus der Anfrage; die Varianten-Liste
|
||||
// bekommt später einen eigenen Editor.
|
||||
return RobustnessAxis.Semantic(new[] { new InputVariant("v1", RequestMessage) });
|
||||
|
||||
default:
|
||||
return new ModelComparisonAxis();
|
||||
}
|
||||
}
|
||||
|
||||
private static List<string> SplitList(string text)
|
||||
=> (text ?? "")
|
||||
.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(s => s.Trim())
|
||||
.Where(s => s.Length > 0)
|
||||
.ToList();
|
||||
|
||||
/// <summary>Parst z.B. "50k, 100k, 150k" zu Zeichenanzahlen (wie im AiTokenAnalyzer).</summary>
|
||||
private static List<int> ParseSizes(string input)
|
||||
{
|
||||
var result = new List<int>();
|
||||
|
||||
foreach (var part in SplitList(input))
|
||||
{
|
||||
var s = part.ToLowerInvariant().Replace(".", "").Replace(" ", "");
|
||||
var factor = 1;
|
||||
|
||||
if (s.EndsWith("k")) { factor = 1000; s = s.Substring(0, s.Length - 1); }
|
||||
else if (s.EndsWith("m")) { factor = 1000000; s = s.Substring(0, s.Length - 1); }
|
||||
|
||||
if (!int.TryParse(s, out var value) || value <= 0)
|
||||
throw new FormatException($"Ungültige Zeichenanzahl: '{part}'");
|
||||
|
||||
result.Add(value * factor);
|
||||
}
|
||||
|
||||
if (result.Count == 0)
|
||||
throw new FormatException("Bitte mindestens eine Zeichenanzahl angeben (z.B. '50k, 100k').");
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Windows.Input;
|
||||
|
||||
namespace AiResponseValidator.ViewModels
|
||||
{
|
||||
/// <summary>Einfaches ICommand für synchrone Aktionen.</summary>
|
||||
public sealed class RelayCommand : ICommand
|
||||
{
|
||||
private readonly Action _Execute;
|
||||
private readonly Func<bool> _CanExecute;
|
||||
|
||||
public RelayCommand(Action execute, Func<bool> canExecute = null)
|
||||
{
|
||||
_Execute = execute ?? throw new ArgumentNullException(nameof(execute));
|
||||
_CanExecute = canExecute;
|
||||
}
|
||||
|
||||
public bool CanExecute(object parameter) => _CanExecute == null || _CanExecute();
|
||||
|
||||
public void Execute(object parameter) => _Execute();
|
||||
|
||||
public event EventHandler CanExecuteChanged
|
||||
{
|
||||
add { CommandManager.RequerySuggested += value; }
|
||||
remove { CommandManager.RequerySuggested -= value; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AiResponseValidator.ViewModels
|
||||
{
|
||||
/// <summary>Die vom Toolbar-Regler wählbaren Szenarien (je eine Achse).</summary>
|
||||
public enum ScenarioKind
|
||||
{
|
||||
RobustheitKontext,
|
||||
RobustheitSemantik,
|
||||
Konsistenz,
|
||||
Modellvergleich
|
||||
}
|
||||
|
||||
/// <summary>Regeltyp für den einfachen Ein-Anfrage-Editor des Grundgerüsts.</summary>
|
||||
public enum RuleKind
|
||||
{
|
||||
Contains,
|
||||
NotContains,
|
||||
Exact,
|
||||
Regex,
|
||||
Canary,
|
||||
Json
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.ComponentModel;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace AiResponseValidator.ViewModels
|
||||
{
|
||||
/// <summary>Minimale MVVM-Basis (manuelles INotifyPropertyChanged, wie im AiTokenAnalyzer).</summary>
|
||||
public abstract class ViewModelBase : INotifyPropertyChanged
|
||||
{
|
||||
public event PropertyChangedEventHandler PropertyChanged;
|
||||
|
||||
protected bool Set<T>(ref T field, T value, [CallerMemberName] string name = null)
|
||||
{
|
||||
if (Equals(field, value))
|
||||
return false;
|
||||
|
||||
field = value;
|
||||
Raise(name);
|
||||
return true;
|
||||
}
|
||||
|
||||
protected void Raise([CallerMemberName] string name = null)
|
||||
=> PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name));
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user