mirror of
https://github.com/tiennm99/ccs.git
synced 2026-10-14 08:18:04 +00:00
refactor(glmt)!: simplify with keyword-based thinking control (v3.4.3)
YAGNI/KISS/DRY refactor removing 926 LOC of unused complexity: - Replace budget-calculator + task-classifier with 40 LOC keyword detection - Add 4-tier thinking: think < think hard < think harder < ultrathink - Remove env vars: CCS_GLMT_THINKING_BUDGET, CCS_GLMT_STREAMING, CCS_GLMT_FORCE_ENGLISH - Add streaming auto-fallback on error - Rename CCS_DEBUG_LOG → CCS_DEBUG (backward compatible) - Fix ultrathink effort: high → max - Fix GLMT proxy path after restructure Test coverage: 27 → 35 tests, all passing Net change: +251, -1177 lines (-926 LOC, 78% reduction) BREAKING CHANGE: Removed CCS_GLMT_THINKING_BUDGET, CCS_GLMT_STREAMING, CCS_GLMT_FORCE_ENGLISH env vars
This commit is contained in:
kaitranntt committed
1 parent
9b9182d418
commit
d3e6be171f
16 files changed
+251
-1177
No files matched your search
@@ -1,338 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* BudgetCalculator Unit Tests
|
||||
*
|
||||
* Tests 4 scenarios:
|
||||
* 1. Default budget (8192) → Thinking enabled for reasoning tasks
|
||||
* 2. Low budget (2048) → Thinking disabled (fast execution)
|
||||
* 3. High budget (16384) → Thinking always enabled
|
||||
* 4. Unlimited (0) → Thinking always enabled
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const BudgetCalculator = require('../../../bin/glmt/budget-calculator');
|
||||
|
||||
describe('BudgetCalculator', () => {
|
||||
describe('Scenario 1: Default budget (8192) - Task-aware thinking', () => {
|
||||
it('should enable thinking for reasoning tasks with default budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', null);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should disable thinking for execution tasks with default budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('execution', null);
|
||||
|
||||
assert.strictEqual(result, false);
|
||||
});
|
||||
|
||||
it('should enable thinking for mixed tasks with default budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('mixed', null);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should use default budget (8192) when not specified', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget = calculator._parseBudget(null);
|
||||
|
||||
assert.strictEqual(budget, 8192);
|
||||
});
|
||||
|
||||
it('should describe default budget correctly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const description = calculator.getBudgetDescription(8192);
|
||||
|
||||
assert.strictEqual(description, 'medium (task-aware thinking)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario 2: Low budget (2048) - Fast execution, no thinking', () => {
|
||||
it('should disable thinking for reasoning tasks with low budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', 2048);
|
||||
|
||||
assert.strictEqual(result, false);
|
||||
});
|
||||
|
||||
it('should disable thinking for execution tasks with low budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('execution', 2048);
|
||||
|
||||
assert.strictEqual(result, false);
|
||||
});
|
||||
|
||||
it('should disable thinking for mixed tasks with low budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('mixed', 2048);
|
||||
|
||||
assert.strictEqual(result, false);
|
||||
});
|
||||
|
||||
it('should parse low budget from string', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget = calculator._parseBudget('2048');
|
||||
|
||||
assert.strictEqual(budget, 2048);
|
||||
});
|
||||
|
||||
it('should describe low budget correctly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const description = calculator.getBudgetDescription(2048);
|
||||
|
||||
assert.strictEqual(description, 'low (fast execution, no thinking)');
|
||||
});
|
||||
|
||||
it('should treat budget <= 2048 as low budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
assert.strictEqual(calculator.shouldEnableThinking('reasoning', 1024), false);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('reasoning', 2000), false);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('reasoning', 2048), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario 3: High budget (16384) - Always enable thinking', () => {
|
||||
it('should enable thinking for reasoning tasks with high budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', 16384);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should enable thinking for execution tasks with high budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('execution', 16384);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should enable thinking for mixed tasks with high budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('mixed', 16384);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should parse high budget from string', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget = calculator._parseBudget('16384');
|
||||
|
||||
assert.strictEqual(budget, 16384);
|
||||
});
|
||||
|
||||
it('should describe high budget correctly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const description = calculator.getBudgetDescription(16384);
|
||||
|
||||
assert.strictEqual(description, 'high (always think)');
|
||||
});
|
||||
|
||||
it('should treat budget > 8192 as high budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', 8193), true);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', 10000), true);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', 32768), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario 4: Unlimited budget (0) - Always enable thinking', () => {
|
||||
it('should enable thinking for reasoning tasks with unlimited budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', 0);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should enable thinking for execution tasks with unlimited budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
// FIXED: _parseBudget(0) now correctly returns 0 (unlimited)
|
||||
const result = calculator.shouldEnableThinking('execution', 0);
|
||||
|
||||
// Unlimited budget should always enable thinking
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should enable thinking for mixed tasks with unlimited budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('mixed', 0);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should parse unlimited from string "unlimited"', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget = calculator._parseBudget('unlimited');
|
||||
|
||||
assert.strictEqual(budget, 0);
|
||||
});
|
||||
|
||||
it('should parse unlimited from string "UNLIMITED" (case insensitive)', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget = calculator._parseBudget('UNLIMITED');
|
||||
|
||||
assert.strictEqual(budget, 0);
|
||||
});
|
||||
|
||||
it('should parse unlimited from number 0', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
// FIXED: _parseBudget(0) now correctly returns 0 (unlimited)
|
||||
const budget = calculator._parseBudget(0);
|
||||
|
||||
// Should return 0 (unlimited)
|
||||
assert.strictEqual(budget, 0);
|
||||
});
|
||||
|
||||
it('should describe unlimited budget correctly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const description = calculator.getBudgetDescription(0);
|
||||
|
||||
assert.strictEqual(description, 'unlimited (always think)');
|
||||
});
|
||||
|
||||
it('should treat negative numbers as unlimited', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget1 = calculator._parseBudget(-1);
|
||||
const budget2 = calculator._parseBudget(-100);
|
||||
|
||||
assert.strictEqual(budget1, 0);
|
||||
assert.strictEqual(budget2, 0);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', -1), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge cases and boundary conditions', () => {
|
||||
it('should handle medium budget boundaries (2049-8192)', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
// Just above low threshold
|
||||
assert.strictEqual(calculator.shouldEnableThinking('reasoning', 2049), true);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', 2049), false);
|
||||
|
||||
// At medium threshold
|
||||
assert.strictEqual(calculator.shouldEnableThinking('reasoning', 8192), true);
|
||||
assert.strictEqual(calculator.shouldEnableThinking('execution', 8192), false);
|
||||
});
|
||||
|
||||
it('should handle invalid budget strings gracefully', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const budget1 = calculator._parseBudget('invalid');
|
||||
const budget2 = calculator._parseBudget('abc123');
|
||||
const budget3 = calculator._parseBudget('');
|
||||
|
||||
assert.strictEqual(budget1, 8192); // Default
|
||||
assert.strictEqual(budget2, 8192); // Default
|
||||
assert.strictEqual(budget3, 8192); // Default
|
||||
});
|
||||
|
||||
it('should handle custom default budget', () => {
|
||||
const calculator = new BudgetCalculator({ defaultBudget: 4096 });
|
||||
|
||||
const budget = calculator._parseBudget(null);
|
||||
|
||||
assert.strictEqual(budget, 4096);
|
||||
});
|
||||
|
||||
it('should handle undefined task type as mixed', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result1 = calculator.shouldEnableThinking(undefined, 8192);
|
||||
const result2 = calculator.shouldEnableThinking(null, 8192);
|
||||
|
||||
// Should default to safe mode (true for medium budget)
|
||||
assert.strictEqual(result1, true);
|
||||
assert.strictEqual(result2, true);
|
||||
});
|
||||
|
||||
it('should handle number type budgets directly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result1 = calculator.shouldEnableThinking('execution', 16384);
|
||||
const result2 = calculator.shouldEnableThinking('execution', 2048);
|
||||
|
||||
assert.strictEqual(result1, true); // High budget
|
||||
assert.strictEqual(result2, false); // Low budget
|
||||
});
|
||||
|
||||
it('should describe all budget ranges correctly', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
assert.strictEqual(calculator.getBudgetDescription(0), 'unlimited (always think)');
|
||||
assert.strictEqual(calculator.getBudgetDescription(1024), 'low (fast execution, no thinking)');
|
||||
assert.strictEqual(calculator.getBudgetDescription(2048), 'low (fast execution, no thinking)');
|
||||
assert.strictEqual(calculator.getBudgetDescription(4096), 'medium (task-aware thinking)');
|
||||
assert.strictEqual(calculator.getBudgetDescription(8192), 'medium (task-aware thinking)');
|
||||
assert.strictEqual(calculator.getBudgetDescription(16384), 'high (always think)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Real-world scenarios', () => {
|
||||
it('should handle planning task with default budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', process.env.CCS_GLMT_THINKING_BUDGET);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
|
||||
it('should handle quick fix task with low budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('execution', 1024);
|
||||
|
||||
assert.strictEqual(result, false);
|
||||
});
|
||||
|
||||
it('should handle complex analysis with high budget', () => {
|
||||
const calculator = new BudgetCalculator();
|
||||
|
||||
const result = calculator.shouldEnableThinking('reasoning', 32768);
|
||||
|
||||
assert.strictEqual(result, true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Run tests if executed directly
|
||||
if (require.main === module) {
|
||||
const Mocha = require('mocha');
|
||||
const mocha = new Mocha({ reporter: 'spec' });
|
||||
mocha.suite.emit('pre-require', global, null, mocha);
|
||||
|
||||
// Load this test file
|
||||
require(module.filename);
|
||||
|
||||
mocha.run(failures => {
|
||||
process.exitCode = failures ? 1 : 0;
|
||||
});
|
||||
}
|
||||
@@ -511,6 +511,100 @@ runner.test('handles invalid thinking type gracefully', () => {
|
||||
assertExists(thinkingConfig, 'thinkingConfig should exist');
|
||||
});
|
||||
|
||||
// Test 28: Keyword detection - "think"
|
||||
runner.test('detects "think" keyword (low effort)', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'think about the solution' }
|
||||
]);
|
||||
|
||||
assertEqual(result.thinking, true, 'thinking should be enabled');
|
||||
assertEqual(result.effort, 'low', 'effort should be low');
|
||||
assertEqual(result.keyword, 'think', 'keyword should be "think"');
|
||||
});
|
||||
|
||||
// Test 29: Keyword detection - "think hard"
|
||||
runner.test('detects "think hard" keyword (medium effort)', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'think hard about edge cases' }
|
||||
]);
|
||||
|
||||
assertEqual(result.thinking, true, 'thinking should be enabled');
|
||||
assertEqual(result.effort, 'medium', 'effort should be medium');
|
||||
assertEqual(result.keyword, 'think hard', 'keyword should be "think hard"');
|
||||
});
|
||||
|
||||
// Test 30: Keyword detection - "think harder"
|
||||
runner.test('detects "think harder" keyword (high effort)', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'think harder about edge cases' }
|
||||
]);
|
||||
|
||||
assertEqual(result.thinking, true, 'thinking should be enabled');
|
||||
assertEqual(result.effort, 'high', 'effort should be high');
|
||||
assertEqual(result.keyword, 'think harder', 'keyword should be "think harder"');
|
||||
});
|
||||
|
||||
// Test 31: Keyword detection - "ultrathink"
|
||||
runner.test('detects "ultrathink" keyword (max effort)', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'ultrathink this complex problem' }
|
||||
]);
|
||||
|
||||
assertEqual(result.thinking, true, 'thinking should be enabled');
|
||||
assertEqual(result.effort, 'max', 'effort should be max');
|
||||
assertEqual(result.keyword, 'ultrathink', 'keyword should be "ultrathink"');
|
||||
});
|
||||
|
||||
// Test 32: Keyword detection - ignores "thinking" (not exact match)
|
||||
runner.test('ignores "thinking" word (not exact match)', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'I am thinking about the solution' }
|
||||
]);
|
||||
|
||||
assertEqual(result, null, 'should return null for non-exact match');
|
||||
});
|
||||
|
||||
// Test 33: Keyword detection - returns null when no keywords
|
||||
runner.test('returns null when no keywords present', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'fix the bug quickly' }
|
||||
]);
|
||||
|
||||
assertEqual(result, null, 'should return null when no keywords found');
|
||||
});
|
||||
|
||||
// Test 34: Keyword priority - ultrathink wins when multiple keywords present
|
||||
runner.test('ultrathink has highest priority when multiple keywords present', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const result = transformer._detectThinkKeywords([
|
||||
{ role: 'user', content: 'think hard and think harder, or maybe ultrathink about this' }
|
||||
]);
|
||||
|
||||
assertEqual(result.thinking, true, 'thinking should be enabled');
|
||||
assertEqual(result.effort, 'max', 'ultrathink should win with max effort');
|
||||
assertEqual(result.keyword, 'ultrathink', 'keyword should be ultrathink');
|
||||
});
|
||||
|
||||
// Test 35: Keyword detection integrates with transformRequest
|
||||
runner.test('keyword detection triggers thinking in transformRequest', () => {
|
||||
const transformer = new GlmtTransformer();
|
||||
const input = {
|
||||
model: 'claude-sonnet-4.5',
|
||||
messages: [{ role: 'user', content: 'think hard about this architecture problem' }]
|
||||
};
|
||||
|
||||
const { thinkingConfig } = transformer.transformRequest(input);
|
||||
|
||||
assertEqual(thinkingConfig.thinking, true, 'keyword should enable thinking');
|
||||
assertEqual(thinkingConfig.effort, 'medium', 'keyword should set medium effort');
|
||||
});
|
||||
|
||||
// Run tests
|
||||
runner.run().then(success => {
|
||||
process.exit(success ? 0 : 1);
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* TaskClassifier Unit Tests
|
||||
*
|
||||
* Tests 3 scenarios:
|
||||
* 1. Reasoning prompt ("plan architecture") → 'reasoning' classification
|
||||
* 2. Execution prompt ("fix bug") → 'execution' classification
|
||||
* 3. Mixed prompt ("analyze and fix") → 'mixed' classification
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const TaskClassifier = require('../../../bin/glmt/task-classifier');
|
||||
|
||||
describe('TaskClassifier', () => {
|
||||
describe('Scenario 1: Reasoning tasks', () => {
|
||||
it('should classify "plan architecture" as reasoning', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Plan a microservices architecture' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should classify "design system" as reasoning', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Design a database schema for e-commerce' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should classify "analyze performance" as reasoning', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Analyze the performance bottlenecks' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should detect multiple reasoning keywords', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Evaluate different approaches and recommend the best strategy' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should classify research tasks as reasoning', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Research best practices for API authentication' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should handle case-insensitive reasoning keywords', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'PLAN THE ARCHITECTURE' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should detect reasoning in array content', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Consider the pros and cons of GraphQL vs REST' }
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario 2: Execution tasks', () => {
|
||||
it('should classify "fix bug" as execution', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Fix the bug in login.js' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should classify "implement feature" as execution', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Implement user authentication' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should classify "debug issue" as execution', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Debug the memory leak in worker.js' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should classify "refactor code" as execution', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Refactor the database queries' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should detect multiple execution keywords', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Add validation and update the form component' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should handle case-insensitive execution keywords', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'FIX THE BUG IN AUTH MODULE' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should classify test tasks as execution', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Run the integration tests' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
|
||||
it('should detect execution in array content', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Create a new API endpoint for users' }
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'execution');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scenario 3: Mixed or ambiguous tasks', () => {
|
||||
it('should classify "analyze and fix" as mixed (tied scores)', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Analyze the issue and fix it' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should classify tasks with equal reasoning and execution keywords as mixed', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Design the API structure and implement it' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should classify tasks with no keywords as mixed', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Help me with the code' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should classify empty content as mixed', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: '' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should return mixed for empty messages array', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should return mixed when no user messages exist', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Hello!' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
|
||||
it('should handle ambiguous prompts', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'What should I do about the authentication?' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'mixed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyWithDetails method', () => {
|
||||
it('should return detailed classification for reasoning task', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Plan and design the system architecture' }
|
||||
];
|
||||
|
||||
const result = classifier.classifyWithDetails(messages);
|
||||
|
||||
assert.strictEqual(result.type, 'reasoning');
|
||||
assert.ok(result.reasoningScore > 0);
|
||||
assert.ok(result.reasoningScore > result.executionScore);
|
||||
assert.ok(result.textLength > 0);
|
||||
assert.ok(result.textPreview.includes('plan'));
|
||||
});
|
||||
|
||||
it('should return detailed classification for execution task', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Fix the bug and run tests' }
|
||||
];
|
||||
|
||||
const result = classifier.classifyWithDetails(messages);
|
||||
|
||||
assert.strictEqual(result.type, 'execution');
|
||||
assert.ok(result.executionScore > 0);
|
||||
assert.ok(result.executionScore > result.reasoningScore);
|
||||
assert.ok(result.textLength > 0);
|
||||
});
|
||||
|
||||
it('should show scores for mixed task', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Evaluate the options and implement the best one' }
|
||||
];
|
||||
|
||||
const result = classifier.classifyWithDetails(messages);
|
||||
|
||||
assert.strictEqual(result.type, 'mixed');
|
||||
assert.strictEqual(result.reasoningScore, result.executionScore);
|
||||
});
|
||||
|
||||
it('should truncate long text in preview', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const longText = 'a'.repeat(200);
|
||||
const messages = [
|
||||
{ role: 'user', content: longText }
|
||||
];
|
||||
|
||||
const result = classifier.classifyWithDetails(messages);
|
||||
|
||||
assert.strictEqual(result.textPreview.length, 103); // 100 + '...'
|
||||
assert.ok(result.textPreview.endsWith('...'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge cases and special scenarios', () => {
|
||||
it('should handle multiple user messages', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Plan the architecture' },
|
||||
{ role: 'assistant', content: 'Here is a plan...' },
|
||||
{ role: 'user', content: 'Implement it' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
// ACTUAL BEHAVIOR: Combines both user messages: "plan the architecture implement it"
|
||||
// "plan" matches reasoning keyword, "implement" matches execution keyword
|
||||
// Score: reasoning=2 (plan, architecture), execution=1 (implement)
|
||||
// Result: reasoning wins
|
||||
assert.strictEqual(result, 'reasoning'); // Changed from 'mixed'
|
||||
});
|
||||
|
||||
it('should handle word boundary matching', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Update the replanning module' } // "plan" in "replanning"
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
// Should not match "plan" in "replanning" due to word boundary
|
||||
assert.strictEqual(result, 'execution'); // Only "update" should match
|
||||
});
|
||||
|
||||
it('should handle custom keywords', () => {
|
||||
const classifier = new TaskClassifier({
|
||||
customKeywords: {
|
||||
reasoning: ['brainstorm', 'strategize'],
|
||||
execution: ['deploy', 'ship']
|
||||
}
|
||||
});
|
||||
|
||||
const messages1 = [{ role: 'user', content: 'Brainstorm ideas' }];
|
||||
const messages2 = [{ role: 'user', content: 'Deploy to production' }];
|
||||
|
||||
assert.strictEqual(classifier.classify(messages1), 'reasoning');
|
||||
assert.strictEqual(classifier.classify(messages2), 'execution');
|
||||
});
|
||||
|
||||
it('should extract text from multiple content blocks', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Plan the' },
|
||||
{ type: 'text', text: 'architecture' }
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should ignore non-text content blocks', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image', source: 'data:...' },
|
||||
{ type: 'text', text: 'Analyze this screenshot' }
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning');
|
||||
});
|
||||
|
||||
it('should handle special characters in keywords', () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Think about the pros and cons' }
|
||||
];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, 'reasoning'); // "think about" and "pros and cons" match
|
||||
});
|
||||
});
|
||||
|
||||
describe('Real-world prompts', () => {
|
||||
const testCases = [
|
||||
{ prompt: 'Create a React component for user profile', expected: 'execution' },
|
||||
{ prompt: 'What is the best approach for state management?', expected: 'reasoning' },
|
||||
{ prompt: 'Compare Redux vs MobX', expected: 'reasoning' },
|
||||
{ prompt: 'Add error handling to the API', expected: 'execution' },
|
||||
{ prompt: 'Investigate why the tests are failing', expected: 'reasoning' },
|
||||
{ prompt: 'Optimize database queries', expected: 'execution' },
|
||||
{ prompt: 'Review the security implications', expected: 'reasoning' },
|
||||
{ prompt: 'Build and deploy the application', expected: 'execution' },
|
||||
{ prompt: 'Should I use TypeScript or JavaScript?', expected: 'mixed' },
|
||||
{ prompt: 'Help me understand this code', expected: 'mixed' }
|
||||
];
|
||||
|
||||
testCases.forEach(({ prompt, expected }) => {
|
||||
it(`should classify "${prompt}" as ${expected}`, () => {
|
||||
const classifier = new TaskClassifier();
|
||||
const messages = [{ role: 'user', content: prompt }];
|
||||
|
||||
const result = classifier.classify(messages);
|
||||
|
||||
assert.strictEqual(result, expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Run tests if executed directly
|
||||
if (require.main === module) {
|
||||
const Mocha = require('mocha');
|
||||
const mocha = new Mocha({ reporter: 'spec' });
|
||||
mocha.suite.emit('pre-require', global, null, mocha);
|
||||
|
||||
// Load this test file
|
||||
require(module.filename);
|
||||
|
||||
mocha.run(failures => {
|
||||
process.exitCode = failures ? 1 : 0;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user