Add source-patched syntax profiles
This commit is contained in:
slhx agent
+196
@@ -0,0 +1,196 @@
|
||||
const std = @import("std");
|
||||
|
||||
// Source-patched syntax profile registry for v1 coding feedback.
|
||||
// req: coding/002, coding/003, ui/002, governance/002
|
||||
|
||||
test {
|
||||
_ = spansAlloc;
|
||||
}
|
||||
|
||||
pub const Error = error{
|
||||
InvalidLanguage,
|
||||
};
|
||||
|
||||
pub const Class = enum {
|
||||
keyword,
|
||||
builtin,
|
||||
string,
|
||||
comment,
|
||||
number,
|
||||
|
||||
pub fn text(self: Class) []const u8 {
|
||||
return switch (self) {
|
||||
.keyword => "keyword",
|
||||
.builtin => "builtin",
|
||||
.string => "string",
|
||||
.comment => "comment",
|
||||
.number => "number",
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub const Span = struct {
|
||||
start: usize,
|
||||
end: usize,
|
||||
class: Class,
|
||||
};
|
||||
|
||||
pub const Profile = struct {
|
||||
language: []const u8,
|
||||
source_patched: bool,
|
||||
};
|
||||
|
||||
pub fn profile(language: []const u8) ?Profile {
|
||||
if (std.mem.eql(u8, language, "zig")) return .{ .language = "zig", .source_patched = true };
|
||||
if (std.mem.eql(u8, language, "text")) return .{ .language = "text", .source_patched = false };
|
||||
return null;
|
||||
}
|
||||
|
||||
pub fn spansAlloc(allocator: std.mem.Allocator, language: []const u8, source: []const u8) ![]Span {
|
||||
const resolved = profile(language) orelse return Error.InvalidLanguage;
|
||||
if (!resolved.source_patched) return allocator.alloc(Span, 0);
|
||||
var spans = std.ArrayList(Span).empty;
|
||||
errdefer spans.deinit(allocator);
|
||||
var i: usize = 0;
|
||||
while (i < source.len) {
|
||||
if (source[i] == '/' and i + 1 < source.len and source[i + 1] == '/') {
|
||||
const start = i;
|
||||
i += 2;
|
||||
while (i < source.len and source[i] != '\n') : (i += 1) {}
|
||||
try spans.append(allocator, .{ .start = start, .end = i, .class = .comment });
|
||||
continue;
|
||||
}
|
||||
if (source[i] == '"') {
|
||||
const start = i;
|
||||
i += 1;
|
||||
while (i < source.len) : (i += 1) {
|
||||
if (source[i] == '\\') {
|
||||
i += @intFromBool(i + 1 < source.len);
|
||||
continue;
|
||||
}
|
||||
if (source[i] == '"') {
|
||||
i += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
try spans.append(allocator, .{ .start = start, .end = i, .class = .string });
|
||||
continue;
|
||||
}
|
||||
if (isDigit(source[i])) {
|
||||
const start = i;
|
||||
while (i < source.len and isDigit(source[i])) : (i += 1) {}
|
||||
try spans.append(allocator, .{ .start = start, .end = i, .class = .number });
|
||||
continue;
|
||||
}
|
||||
if (isIdentStart(source[i])) {
|
||||
const start = i;
|
||||
i += 1;
|
||||
while (i < source.len and isIdentContinue(source[i])) : (i += 1) {}
|
||||
const token = source[start..i];
|
||||
if (keywordClass(token)) |class| try spans.append(allocator, .{ .start = start, .end = i, .class = class });
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return spans.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
pub fn rowsAlloc(allocator: std.mem.Allocator, language: []const u8, source: []const u8) ![][]const u8 {
|
||||
const spans = try spansAlloc(allocator, language, source);
|
||||
defer allocator.free(spans);
|
||||
var rows = std.ArrayList([]const u8).empty;
|
||||
errdefer {
|
||||
for (rows.items) |row| allocator.free(row);
|
||||
rows.deinit(allocator);
|
||||
}
|
||||
const resolved = profile(language) orelse return Error.InvalidLanguage;
|
||||
if (!resolved.source_patched) {
|
||||
try rows.append(allocator, try allocator.dupe(u8, "syntax:text:fallback:no_spans"));
|
||||
return rows.toOwnedSlice(allocator);
|
||||
}
|
||||
if (spans.len == 0) {
|
||||
try rows.append(allocator, try std.fmt.allocPrint(allocator, "syntax:{s}:no_spans", .{resolved.language}));
|
||||
return rows.toOwnedSlice(allocator);
|
||||
}
|
||||
for (spans) |span| {
|
||||
const preview = try previewAlloc(allocator, source[span.start..span.end]);
|
||||
defer allocator.free(preview);
|
||||
try rows.append(allocator, try std.fmt.allocPrint(
|
||||
allocator,
|
||||
"syntax:{s}:{s}:{d}:{d}:{s}",
|
||||
.{ resolved.language, span.class.text(), span.start, span.end, preview },
|
||||
));
|
||||
}
|
||||
return rows.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
fn keywordClass(token: []const u8) ?Class {
|
||||
const keywords = [_][]const u8{ "pub", "const", "var", "fn", "return", "if", "else", "while", "for", "switch", "try", "catch", "defer", "test", "struct", "enum", "error", "union", "comptime", "inline", "break", "continue" };
|
||||
for (keywords) |keyword| if (std.mem.eql(u8, token, keyword)) return .keyword;
|
||||
const builtins = [_][]const u8{ "true", "false", "null", "undefined", "usize", "u8", "i32", "void", "bool", "anyerror" };
|
||||
for (builtins) |builtin| if (std.mem.eql(u8, token, builtin)) return .builtin;
|
||||
return null;
|
||||
}
|
||||
|
||||
fn previewAlloc(allocator: std.mem.Allocator, token: []const u8) ![]u8 {
|
||||
var out = std.ArrayList(u8).empty;
|
||||
errdefer out.deinit(allocator);
|
||||
var i: usize = 0;
|
||||
while (i < token.len and out.items.len < 48) {
|
||||
const len = std.unicode.utf8ByteSequenceLength(token[i]) catch 1;
|
||||
const end = @min(token.len, i + len);
|
||||
const slice = token[i..end];
|
||||
if (slice.len == 1 and (slice[0] <= 0x20 or slice[0] == ':' or slice[0] == '|')) {
|
||||
try out.append(allocator, '_');
|
||||
} else {
|
||||
try out.appendSlice(allocator, slice);
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
if (out.items.len == 0) try out.append(allocator, '_');
|
||||
return out.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
fn isDigit(byte: u8) bool {
|
||||
return byte >= '0' and byte <= '9';
|
||||
}
|
||||
|
||||
fn isIdentStart(byte: u8) bool {
|
||||
return (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z') or byte == '_';
|
||||
}
|
||||
|
||||
fn isIdentContinue(byte: u8) bool {
|
||||
return isIdentStart(byte) or isDigit(byte);
|
||||
}
|
||||
|
||||
fn freeRows(allocator: std.mem.Allocator, rows: []const []const u8) void {
|
||||
for (rows) |row| allocator.free(row);
|
||||
allocator.free(rows);
|
||||
}
|
||||
|
||||
test "regular: zig source profile produces deterministic highlight spans" {
|
||||
const source = "pub const answer = 42; // ok\n";
|
||||
const spans = try spansAlloc(std.testing.allocator, "zig", source);
|
||||
defer std.testing.allocator.free(spans);
|
||||
try std.testing.expectEqual(@as(usize, 4), spans.len);
|
||||
try std.testing.expectEqualDeep(Span{ .start = 0, .end = 3, .class = .keyword }, spans[0]);
|
||||
try std.testing.expectEqualDeep(Span{ .start = 4, .end = 9, .class = .keyword }, spans[1]);
|
||||
try std.testing.expectEqualDeep(Span{ .start = 19, .end = 21, .class = .number }, spans[2]);
|
||||
try std.testing.expectEqualDeep(Span{ .start = 23, .end = 28, .class = .comment }, spans[3]);
|
||||
}
|
||||
|
||||
test "regular: zig syntax rows include class byte ranges and previews" {
|
||||
const rows = try rowsAlloc(std.testing.allocator, "zig", "fn main() void { return; }");
|
||||
defer freeRows(std.testing.allocator, rows);
|
||||
try std.testing.expectEqualStrings("syntax:zig:keyword:0:2:fn", rows[0]);
|
||||
try std.testing.expectEqualStrings("syntax:zig:builtin:10:14:void", rows[1]);
|
||||
try std.testing.expectEqualStrings("syntax:zig:keyword:17:23:return", rows[2]);
|
||||
}
|
||||
|
||||
test "adversarial: plain text falls back and unknown languages fail" {
|
||||
const rows = try rowsAlloc(std.testing.allocator, "text", "pub const not highlighted");
|
||||
defer freeRows(std.testing.allocator, rows);
|
||||
try std.testing.expectEqual(@as(usize, 1), rows.len);
|
||||
try std.testing.expectEqualStrings("syntax:text:fallback:no_spans", rows[0]);
|
||||
try std.testing.expectError(Error.InvalidLanguage, rowsAlloc(std.testing.allocator, "python", "def x(): pass"));
|
||||
}
|
||||
Reference in New Issue
Block a user