| Prompt: | (initial) |
| Time taken: | 0.9 seconds |
| Prompt tokens: | 1.4K |
| Response tokens: | 562 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
using SysprogsDevTools.Tokenizer;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Sysprogs.IDE.Framework.Languages.DiffExport
{
public class ExporterDemo
{
public class LineWrapper
{
public int Number { get; }
public List<TokenWrapper> Tokens { get; }
public LineWrapper(int number, List<TokenWrapper> tokens)
{
Number = number;
Tokens = tokens;
}
}
public class TokenWrapper
{
private readonly UniversalTokenizer.Token _originalToken;
public string Value { get; }
public DiffState State { get; }
public enum DiffState { Normal, Added, Removed }
public TokenWrapper(UniversalTokenizer.Token originalToken, DiffState state)
{
_originalToken = originalToken;
Value = _originalToken.GetText();
State = state;
}
}
public static string DumpTokens(string text, UniversalTokenizer tokenizer)
{
var lines = text.Split('\n');
var wrappedLines = new List<LineWrapper>();
var tokenType = UniversalTokenizer.TokenType.Invalid;
var contextFlags = 0;
for (var i = 0; i < lines.Length; i++)
foreach (var line in lines)
{
var tokens = tokenizer.TokenizeLine(lines[i], ref tokenType, contextFlags);
var wrappedTokens = tokens.Select(t => new TokenWrapper(t, TokenWrapper.DiffState.Normal)).ToList();
wrappedLines.Add(new LineWrapper(i + 1, wrappedTokens));
}
var result = new StringBuilder();
foreach (var line in wrappedLines)
{
foreach (var token in line.Tokens)
{
string cssClass = TranslateTokenType(token.Type);
if (cssClass != null)
result.Append($"<span class=\"{cssClass}\">{token.Value}</span>");
else
result.Append(token.Value);
}
result.AppendLine();
}
return result.ToString();
}
private static string TranslateTokenType(UniversalTokenizer.TokenType tokenType)
{
switch (tokenType)
{
case UniversalTokenizer.TokenType.Keyword:
return "keyword";
case UniversalTokenizer.TokenType.CharacterLiteral:
case UniversalTokenizer.TokenType.StringLiteral:
return "literal";
case UniversalTokenizer.TokenType.Comment:
return "comment";
default:
return null;
}
}
}
}