\d t

C:{1_'(&x="\n")_ x}"
 
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ
.
0123456789
\"
\\
-_~!@#$%^&*=+|<,>?'/{}
()[];
:
`"                                              / character classes

Y:_ci 254                                       / null1
Z:_ci 255                                       / null2
B:**C                                           / blank
A:C 1                                           / alpha
N:C 3                                           / number
M:*C 6                                          / -
U:C[6]1                                         / _
C[1],:Y                                         / null1 is a letter
C[3],:Z                                         / null2 is a digit
X:_ci(!256)_dvl 0,_ic,/1_ C                     / ~ blanks

V:" a0.9nNqQxXstSfFgGcC()"                      / states
W:V?/:" aNXSFGcC("                              / endpoints

T:"
 af0qff(cs
 aaaqff(cs
Nn.0qff(cs
Nnf9qff(cs
Nnf9qff(cs
Naf9qff(cs
Naf0qff(cs
qqqqQxqqcq
 af0qff(cs
XXXXXXXXXX
qqqqqqqqqq
 tfttff(cS
 ttttff(cS
 SSSSff(cs
 aF0qgg(Fs
 ag0qgg(gs
 aG0qff(Gs
 af0qff(fs
 af0qff(Cs
 af0qff(cs
 af0qff)cs
 af0qff(cs"                                     / states x classes

fsm:{{.[x;(;y);:;z]}/[(#x)#,&256;_ic y;+x]}     / finite state machine
S:fsm[V?/:/:1_'(&T="\n")_ T]C                   / transition matrix

bx:_ssr/[;X;B]                                  /    <- X
mz:_ssr[;Z;M]                                   /  - <- Z
zm:_ssr/[;(B,M),/:N;(B,Z),/:N]                  / Zx <- -x
uy:_ssr[;Y;U]                                   /  _ <- Y
yu:_ssr/[;(B,U),/:A;(B,Y),/:A]                  / Yx <- _x
db:{:[#x;x _di&x[;0]=B;x]}                      / delete blank tokens
ct:{(&~0=':W _binl y)_ x}                       / cut tokens
tr:{ct[uy mz x]1_0 S\_ic x}                     / compute state transitions

tokens:db tr yu zm bx B,                        / tokenize

\d ^
tokens:t.tokens