Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 39 additions & 21 deletions tool/lib/src/commands/validate_skill_command.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'dart:io';

import 'package:logging/logging.dart';
import 'package:path/path.dart' as p;
import 'package:yaml/yaml.dart';

import '../models/skill_params.dart';
import '../services/gemini_service.dart';
Expand Down Expand Up @@ -73,29 +74,46 @@ class ValidateSkillCommand extends BaseSkillCommand {

final existingSkillFileContent = existingSkillFile.readAsStringSync();

// Check for verbatim name
final namePattern = RegExp(
'name:\\s*["\']?${RegExp.escape(skill.name)}["\']?',
);
if (!namePattern.hasMatch(existingSkillFileContent)) {
logger.severe(
' Validation Failed: Skill name mismatch in ${existingSkillFile.path}. '
'Expected "name: ${skill.name}" (quotes allowed)',
);
// Check for verbatim name and description
final frontmatterRegex = RegExp(r'^---\s*\n(.*?)\n---', dotAll: true);
final match = frontmatterRegex.firstMatch(existingSkillFileContent);

String generationDate = 'Unknown';
String modelName = 'Unknown';

if (match != null) {
final yamlText = match.group(1)!;
try {
final yaml = loadYaml(yamlText);
if (yaml is Map) {
final name = yaml['name'];
if (name != skill.name) {
logger.severe(
' Validation Failed: Skill name mismatch in ${existingSkillFile.path}. '
'Expected "${skill.name}", got "$name"',
);
}

final metadata = yaml['metadata'];
if (metadata is Map) {
generationDate = metadata['last_modified']?.toString() ?? 'Unknown';
modelName = metadata['model']?.toString() ?? 'Unknown';
}
}
} catch (e) {
logger.warning(' Failed to parse frontmatter as YAML: $e');
}
} else {
logger.warning(' No frontmatter found in ${existingSkillFile.path}');
// Fallback to strict check for safety if there's no frontmatter at all
if (!existingSkillFileContent.contains('name: ${skill.name}')) {
logger.severe(
' Validation Failed: Skill name mismatch in ${existingSkillFile.path}. '
'Expected "name: ${skill.name}"',
);
}
}

// Extract metadata from existing content
final generationDate =
RegExp(
'last_modified: (.*)',
).firstMatch(existingSkillFileContent)?.group(1) ??
'Unknown';
final modelName =
RegExp(
'model: (.*)',
).firstMatch(existingSkillFileContent)?.group(1) ??
'Unknown';

// Compare
final dryRun = argResults?['dry-run'] as bool? ?? false;
if (dryRun) {
Expand Down
75 changes: 75 additions & 0 deletions tool/test/validate_skills_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -730,5 +730,80 @@ Content
final valDir = Directory(p.join(validationDir.path, skillName));
expect(valDir.existsSync(), isFalse);
});

test('validates successfully when skill name is quoted in frontmatter', () async {
const skillName = 'quoted-skill';
final skillDir = Directory(p.join(skillsDir.path, skillName));
await skillDir.create();
final skillFile = File(p.join(skillDir.path, 'SKILL.md'));
await skillFile.writeAsString('''
---
name: "$skillName"
description: "Desc"
metadata:
model: "test-model"
last_modified: "date"
---
Content
''');

final configFile = File(p.join(tempDir.path, 'config.yaml'));
await configFile.writeAsString(
jsonEncode([
{
'name': skillName,
'description': 'Desc',
'resources': ['https://example.com/source'],
},
]),
);

final mockClient = MockClient((request) async {
final url = request.url.toString();
if (url == 'https://example.com/source') {
return http.Response('# Source', 200);
}
if (url.contains('generativelanguage')) {
return http.Response(
jsonEncode({
'candidates': [
{
'content': {
'parts': [
{
'text': '# Validation Report\nGrade: 100',
},
],
},
},
],
}),
200,
);
}
return http.Response('Not Found', 404);
});

runner = CommandRunner<void>('skills', 'Test runner')
..addCommand(
ValidateSkillCommand(
environment: {'GEMINI_API_KEY': 'test-key'},
outputDir: skillsDir,
validationDir: validationDir,
httpClient: mockClient,
),
);

await IOOverrides.runZoned(() async {
await runner.run(['validate-skill', configFile.path]);
}, getCurrentDirectory: () => tempDir);

expect(logs, contains('Validating skill: $skillName...'));
expect(
logs,
isNot(contains(contains('Validation Failed: Skill name mismatch'))),
);
expect(logs, contains(contains('Validation report written to')));
});
});
}
Loading