Quick Start

npm install @kyro/judge

steps:
  - name: relevance
    prompt: |
      You are evaluating whether an AI assistant's response is relevant to the user's question.
 
      User question: {{input.user_message}}
      Assistant response: {{input.assistant_message}}
 
      Respond with SUCCESS if the response is relevant, ERROR if it is not.
      Provide a brief root_cause explaining your decision.
    output_format: json

import { Judge } from '@kyro/judge';
import { ProviderFactory } from '@kyro/shared';
 
const provider = ProviderFactory.create({
  provider: 'openai',
  model: 'gpt-4o',
  apiKey: process.env.OPENAI_API_KEY,
});
 
const judge = new Judge('./pipeline.yaml', provider);
 
const result = await judge.run({
  user_message: 'What is the capital of France?',
  assistant_message: 'The capital of France is Paris.',
});
 
console.log(result);

{
  "status": "SUCCESS",
  "message": "All judges passed",
  "data": {
    "relevance": {
      "status": "SUCCESS",
      "root_cause": "The response directly and accurately answers the user's question.",
      "thinking": "The user asked about the capital of France. The assistant responded with 'Paris', which is correct and directly relevant.",
      "tokens": { "input": 142, "output": 38 }
    }
  },
  "input": { "type": "object", "size": 92 }
}