Grammalecte  Hex Artifact Content

Artifact 5f94dc04ead495f0a2c24e74705487939191b7461f9c287da46149534362b471:


0000: 2f 2f 20 4a 61 76 61 53 63 72 69 70 74 0a 2f 2f  // JavaScript.//
0010: 20 56 65 72 79 20 73 69 6d 70 6c 65 20 74 6f 6b   Very simple tok
0020: 65 6e 69 7a 65 72 0a 2f 2a 6a 73 6c 69 6e 74 20  enizer./*jslint 
0030: 65 73 76 65 72 73 69 6f 6e 3a 20 36 2a 2f 0a 2f  esversion: 6*/./
0040: 2a 67 6c 6f 62 61 6c 20 72 65 71 75 69 72 65 2c  *global require,
0050: 65 78 70 6f 72 74 73 2a 2f 0a 0a 22 75 73 65 20  exports*/.."use 
0060: 73 74 72 69 63 74 22 3b 0a 0a 0a 69 66 20 28 74  strict";...if (t
0070: 79 70 65 6f 66 28 72 65 71 75 69 72 65 29 20 21  ypeof(require) !
0080: 3d 3d 20 27 75 6e 64 65 66 69 6e 65 64 27 29 20  == 'undefined') 
0090: 7b 0a 20 20 20 20 76 61 72 20 68 65 6c 70 65 72  {.    var helper
00a0: 73 20 3d 20 72 65 71 75 69 72 65 28 22 72 65 73  s = require("res
00b0: 6f 75 72 63 65 3a 2f 2f 67 72 61 6d 6d 61 6c 65  ource://grammale
00c0: 63 74 65 2f 67 72 61 70 68 73 70 65 6c 6c 2f 68  cte/graphspell/h
00d0: 65 6c 70 65 72 73 2e 6a 73 22 29 3b 0a 7d 0a 0a  elpers.js");.}..
00e0: 0a 63 6f 6e 73 74 20 61 54 6b 7a 50 61 74 74 65  .const aTkzPatte
00f0: 72 6e 73 20 3d 20 7b 0a 20 20 20 20 2f 2f 20 41  rns = {.    // A
0100: 6c 6c 20 72 65 67 65 78 70 73 20 6d 75 73 74 20  ll regexps must 
0110: 73 74 61 72 74 20 77 69 74 68 20 5e 2e 0a 20 20  start with ^..  
0120: 20 20 22 64 65 66 61 75 6c 74 22 3a 0a 20 20 20    "default":.   
0130: 20 20 20 20 20 5b 0a 20 20 20 20 20 20 20 20 20       [.         
0140: 20 20 20 5b 2f 5e 5b 20 c2 a0 e2 80 af 5c 74 5d     [/^[ .....\t]
0150: 2b 2f 2c 20 27 53 50 41 43 45 27 5d 2c 0a 20 20  +/, 'SPACE'],.  
0160: 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5c 2f 28            [/^\/(
0170: 3f 3a 7e 7c 62 69 6e 7c 62 6f 6f 74 7c 64 65 76  ?:~|bin|boot|dev
0180: 7c 65 74 63 7c 68 6f 6d 65 7c 6c 69 62 7c 6d 6e  |etc|home|lib|mn
0190: 74 7c 6f 70 74 7c 72 6f 6f 74 7c 73 62 69 6e 7c  t|opt|root|sbin|
01a0: 74 6d 70 7c 75 73 72 7c 76 61 72 7c 42 75 72 65  tmp|usr|var|Bure
01b0: 61 75 7c 44 6f 63 75 6d 65 6e 74 73 7c 49 6d 61  au|Documents|Ima
01c0: 67 65 73 7c 4d 75 73 69 71 75 65 7c 50 75 62 6c  ges|Musique|Publ
01d0: 69 63 7c 54 c3 a9 6c c3 a9 63 68 61 72 67 65 6d  ic|T..l..chargem
01e0: 65 6e 74 73 7c 56 69 64 c3 a9 6f 73 29 28 3f 3a  ents|Vid..os)(?:
01f0: 5c 2f 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80  \/[a-zA-Z..-....
0200: 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f  -..0-9..-....-..
0210: c4 80 2d ca af ef ac 81 2d ef ac 86 5f 2e 28 29  ..-.....-..._.()
0220: 2d 5d 2b 29 2a 2f 2c 20 27 46 4f 4c 44 45 52 55  -]+)*/, 'FOLDERU
0230: 4e 49 58 27 5d 2c 0a 20 20 20 20 20 20 20 20 20  NIX'],.         
0240: 20 20 20 5b 2f 5e 5b 61 2d 7a 41 2d 5a 5d 3a 5c     [/^[a-zA-Z]:\
0250: 5c 28 3f 3a 50 72 6f 67 72 61 6d 20 46 69 6c 65  \(?:Program File
0260: 73 28 3f 3a 20 5c 28 78 38 36 5c 29 7c 29 7c 5b  s(?: \(x86\)|)|[
0270: 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96  a-zA-Z..-....-..
0280: 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d  0-9..-....-....-
0290: ca af ef ac 81 2d ef ac 86 2e 28 29 5d 2b 29 28  .....-....()]+)(
02a0: 3f 3a 5c 5c 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6  ?:\\[a-zA-Z..-..
02b0: c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d  ..-..0-9..-....-
02c0: c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5f 2e  ....-.....-..._.
02d0: 28 29 2d 5d 2b 29 2a 2f 2c 20 27 46 4f 4c 44 45  ()-]+)*/, 'FOLDE
02e0: 52 57 49 4e 27 5d 2c 0a 20 20 20 20 20 20 20 20  RWIN'],.        
02f0: 20 20 20 20 5b 2f 5e 5b 2c 2e 3b 3a 21 3f e2 80      [/^[,.;:!?..
0300: a6 c2 ab c2 bb e2 80 9c e2 80 9d e2 80 98 e2 80  ................
0310: 99 22 28 29 7b 7d 5c 5b 5c 5d c2 b7 e2 80 93 e2  ."(){}\[\]......
0320: 80 94 5d 2f 2c 20 27 53 45 50 41 52 41 54 4f 52  ..]/, 'SEPARATOR
0330: 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20  '],.            
0340: 5b 2f 5e 5b 41 2d 5a 5d 5b 2e 5d 5b 41 2d 5a 5d  [/^[A-Z][.][A-Z]
0350: 5b 2e 5d 28 3f 3a 5b 41 2d 5a 5d 5b 2e 5d 29 2a  [.](?:[A-Z][.])*
0360: 2f 2c 20 27 41 43 52 4f 4e 59 4d 27 5d 2c 0a 20  /, 'ACRONYM'],. 
0370: 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 28 3f             [/^(?
0380: 3a 68 74 74 70 73 3f 3a 5c 2f 5c 2f 7c 77 77 77  :https?:\/\/|www
0390: 5b 2e 5d 7c 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6  [.]|[a-zA-Z..-..
03a0: c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d  ..-..0-9..-....-
03b0: c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5f 2d  ....-.....-..._-
03c0: 5d 2b 5b 40 2e 5d 5b 61 2d 7a 41 2d 5a c3 a0 2d  ]+[@.][a-zA-Z..-
03d0: c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3  ....-..0-9..-...
03e0: 98 2d c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86  .-....-.....-...
03f0: 5f 2d 5d 7b 32 2c 7d 5b 40 2e 5d 29 5b 61 2d 7a  _-]{2,}[@.])[a-z
0400: 41 2d 5a 30 2d 39 5d 5b 61 2d 7a 41 2d 5a c3 a0  A-Z0-9][a-zA-Z..
0410: 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf  -....-..0-9..-..
0420: c3 98 2d c3 9f c4 80 2d ca af ef ac 81 2d ef ac  ..-....-.....-..
0430: 86 5f 2e 5c 2f 3f 26 21 25 3d 2b 2a 22 27 40 24  ._.\/?&!%=+*"'@$
0440: 23 2d 5d 2b 2f 2c 20 27 4c 49 4e 4b 27 5d 2c 0a  #-]+/, 'LINK'],.
0450: 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5b              [/^[
0460: 23 40 5d 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3  #@][a-zA-Z..-...
0470: 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3  .-..0-9..-....-.
0480: 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5f 2d 5d  ...-.....-..._-]
0490: 2b 2f 2c 20 27 54 41 47 27 5d 2c 0a 20 20 20 20  +/, 'TAG'],.    
04a0: 20 20 20 20 20 20 20 20 5b 2f 5e 3c 5b 61 2d 7a          [/^<[a-z
04b0: 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d 39  A-Z..-....-..0-9
04c0: c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d ca af ef  ..-....-....-...
04d0: ac 81 2d ef ac 86 5d 2b 2e 2a 3f 3e 7c 3c 5c 2f  ..-...]+.*?>|<\/
04e0: 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3  [a-zA-Z..-....-.
04f0: 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4 80  .0-9..-....-....
0500: 2d ca af ef ac 81 2d ef ac 86 5d 2b 20 2a 3e 2f  -.....-...]+ *>/
0510: 2c 20 27 48 54 4d 4c 27 5d 2c 0a 20 20 20 20 20  , 'HTML'],.     
0520: 20 20 20 20 20 20 20 5b 2f 5e 5c 5b 5c 2f 3f 5b         [/^\[\/?[
0530: 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96  a-zA-Z..-....-..
0540: 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d  0-9..-....-....-
0550: ca af ef ac 81 2d ef ac 86 5d 2b 5c 5d 2f 2c 20  .....-...]+\]/, 
0560: 27 50 53 45 55 44 4f 48 54 4d 4c 27 5d 2c 0a 20  'PSEUDOHTML'],. 
0570: 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 26 5c             [/^&\
0580: 77 2b 3b 28 3f 3a 5c 77 2b 3b 7c 29 2f 2c 20 27  w+;(?:\w+;|)/, '
0590: 48 54 4d 4c 45 4e 54 49 54 59 27 5d 2c 0a 20 20  HTMLENTITY'],.  
05a0: 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5c 64 5c            [/^\d\
05b0: 64 3f 68 5c 64 5c 64 5c 62 2f 2c 20 27 48 4f 55  d?h\d\d\b/, 'HOU
05c0: 52 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20  R'],.           
05d0: 20 5b 2f 5e 2d 3f 5c 64 2b 28 3f 3a 5b 2e 2c 5d   [/^-?\d+(?:[.,]
05e0: 5c 64 2b 7c 29 2f 2c 20 27 4e 55 4d 27 5d 2c 0a  \d+|)/, 'NUM'],.
05f0: 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5b              [/^[
0600: 25 e2 80 b0 2b 3d 2a 2f 3c 3e e2 a9 be e2 a9 bd  %...+=*/<>......
0610: 2d 5d 2f 2c 20 27 53 49 47 4e 27 5d 2c 0a 20 20  -]/, 'SIGN'],.  
0620: 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5b 61 2d            [/^[a-
0630: 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d  zA-Z..-....-..0-
0640: 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d ca af  9..-....-....-..
0650: ef ac 81 2d ef ac 86 5d 2b 28 3f 3a 5b e2 80 99  ...-...]+(?:[...
0660: 27 60 2d 5d 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6  '`-][a-zA-Z..-..
0670: c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d  ..-..0-9..-....-
0680: c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5d 2b  ....-.....-...]+
0690: 29 2a 2f 2c 20 27 57 4f 52 44 27 5d 0a 20 20 20  )*/, 'WORD'].   
06a0: 20 20 20 20 20 5d 2c 0a 20 20 20 20 22 66 72 22       ],.    "fr"
06b0: 3a 0a 20 20 20 20 20 20 20 20 5b 0a 20 20 20 20  :.        [.    
06c0: 20 20 20 20 20 20 20 20 5b 2f 5e 5b 20 c2 a0 e2          [/^[ ...
06d0: 80 af 5c 74 5d 2b 2f 2c 20 27 53 50 41 43 45 27  ..\t]+/, 'SPACE'
06e0: 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20 5b  ],.            [
06f0: 2f 5e 5c 2f 28 3f 3a 7e 7c 62 69 6e 7c 62 6f 6f  /^\/(?:~|bin|boo
0700: 74 7c 64 65 76 7c 65 74 63 7c 68 6f 6d 65 7c 6c  t|dev|etc|home|l
0710: 69 62 7c 6d 6e 74 7c 6f 70 74 7c 72 6f 6f 74 7c  ib|mnt|opt|root|
0720: 73 62 69 6e 7c 74 6d 70 7c 75 73 72 7c 76 61 72  sbin|tmp|usr|var
0730: 7c 42 75 72 65 61 75 7c 44 6f 63 75 6d 65 6e 74  |Bureau|Document
0740: 73 7c 49 6d 61 67 65 73 7c 4d 75 73 69 71 75 65  s|Images|Musique
0750: 7c 50 75 62 6c 69 63 7c 54 c3 a9 6c c3 a9 63 68  |Public|T..l..ch
0760: 61 72 67 65 6d 65 6e 74 73 7c 56 69 64 c3 a9 6f  argements|Vid..o
0770: 73 29 28 3f 3a 5c 2f 5b 61 2d 7a 41 2d 5a c3 a0  s)(?:\/[a-zA-Z..
0780: 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf  -....-..0-9..-..
0790: c3 98 2d c3 9f c4 80 2d ca af ef ac 81 2d ef ac  ..-....-.....-..
07a0: 86 5f 2e 28 29 2d 5d 2b 29 2a 2f 2c 20 27 46 4f  ._.()-]+)*/, 'FO
07b0: 4c 44 45 52 55 4e 49 58 27 5d 2c 0a 20 20 20 20  LDERUNIX'],.    
07c0: 20 20 20 20 20 20 20 20 5b 2f 5e 5b 61 2d 7a 41          [/^[a-zA
07d0: 2d 5a 5d 3a 5c 5c 28 3f 3a 50 72 6f 67 72 61 6d  -Z]:\\(?:Program
07e0: 20 46 69 6c 65 73 28 3f 3a 20 5c 28 78 38 36 5c   Files(?: \(x86\
07f0: 29 7c 29 7c 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6  )|)|[a-zA-Z..-..
0800: c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d  ..-..0-9..-....-
0810: c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 2e 28  ....-.....-....(
0820: 29 5d 2b 29 28 3f 3a 5c 5c 5b 61 2d 7a 41 2d 5a  )]+)(?:\\[a-zA-Z
0830: c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d  ..-....-..0-9..-
0840: c3 bf c3 98 2d c3 9f c4 80 2d ca af ef ac 81 2d  ....-....-.....-
0850: ef ac 86 5f 2e 28 29 2d 5d 2b 29 2a 2f 2c 20 27  ..._.()-]+)*/, '
0860: 46 4f 4c 44 45 52 57 49 4e 27 5d 2c 0a 20 20 20  FOLDERWIN'],.   
0870: 20 20 20 20 20 20 20 20 20 5b 2f 5e 5b 2c 2e 3b           [/^[,.;
0880: 3a 21 3f e2 80 a6 c2 ab c2 bb e2 80 9c e2 80 9d  :!?.............
0890: e2 80 98 e2 80 99 22 28 29 7b 7d 5c 5b 5c 5d c2  ......"(){}\[\].
08a0: b7 e2 80 93 e2 80 94 5d 2f 2c 20 27 53 45 50 41  .......]/, 'SEPA
08b0: 52 41 54 4f 52 27 5d 2c 0a 20 20 20 20 20 20 20  RATOR'],.       
08c0: 20 20 20 20 20 5b 2f 5e 5b 41 2d 5a 5d 5b 2e 5d       [/^[A-Z][.]
08d0: 5b 41 2d 5a 5d 5b 2e 5d 28 3f 3a 5b 41 2d 5a 5d  [A-Z][.](?:[A-Z]
08e0: 5b 2e 5d 29 2a 2f 2c 20 27 41 43 52 4f 4e 59 4d  [.])*/, 'ACRONYM
08f0: 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20  '],.            
0900: 5b 2f 5e 28 3f 3a 68 74 74 70 73 3f 3a 5c 2f 5c  [/^(?:https?:\/\
0910: 2f 7c 77 77 77 5b 2e 5d 7c 5b 61 2d 7a 41 2d 5a  /|www[.]|[a-zA-Z
0920: c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d  ..-....-..0-9..-
0930: c3 bf c3 98 2d c3 9f c4 80 2d ca af ef ac 81 2d  ....-....-.....-
0940: ef ac 86 5f 2d 5d 2b 5b 40 2e 5d 5b 61 2d 7a 41  ..._-]+[@.][a-zA
0950: 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3  -Z..-....-..0-9.
0960: b8 2d c3 bf c3 98 2d c3 9f c4 80 2d ca af ef ac  .-....-....-....
0970: 81 2d ef ac 86 5f 2d 5d 7b 32 2c 7d 5b 40 2e 5d  .-..._-]{2,}[@.]
0980: 29 5b 61 2d 7a 41 2d 5a 30 2d 39 5d 5b 61 2d 7a  )[a-zA-Z0-9][a-z
0990: 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96 30 2d 39  A-Z..-....-..0-9
09a0: c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d ca af ef  ..-....-....-...
09b0: ac 81 2d ef ac 86 5f 2e 5c 2f 3f 26 21 25 3d 2b  ..-..._.\/?&!%=+
09c0: 2a 22 27 40 24 23 2d 5d 2b 2f 2c 20 27 4c 49 4e  *"'@$#-]+/, 'LIN
09d0: 4b 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20  K'],.           
09e0: 20 5b 2f 5e 5b 23 40 5d 5b 61 2d 7a 41 2d 5a c3   [/^[#@][a-zA-Z.
09f0: a0 2d c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3  .-....-..0-9..-.
0a00: bf c3 98 2d c3 9f c4 80 2d ca af ef ac 81 2d ef  ...-....-.....-.
0a10: ac 86 5f 2d 5d 2b 2f 2c 20 27 54 41 47 27 5d 2c  .._-]+/, 'TAG'],
0a20: 0a 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e  .            [/^
0a30: 3c 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d  <[a-zA-Z..-....-
0a40: c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4  ..0-9..-....-...
0a50: 80 2d ca af ef ac 81 2d ef ac 86 5d 2b 2e 2a 3f  .-.....-...]+.*?
0a60: 3e 7c 3c 5c 2f 5b 61 2d 7a 41 2d 5a c3 a0 2d c3  >|<\/[a-zA-Z..-.
0a70: b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98  ...-..0-9..-....
0a80: 2d c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5d  -....-.....-...]
0a90: 2b 20 2a 3e 2f 2c 20 27 48 54 4d 4c 27 5d 2c 0a  + *>/, 'HTML'],.
0aa0: 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5c              [/^\
0ab0: 5b 5c 2f 3f 5b 61 2d 7a 41 2d 5a c3 a0 2d c3 b6  [\/?[a-zA-Z..-..
0ac0: c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3 98 2d  ..-..0-9..-....-
0ad0: c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86 5d 2b  ....-.....-...]+
0ae0: 5c 5d 2f 2c 20 27 50 53 45 55 44 4f 48 54 4d 4c  \]/, 'PSEUDOHTML
0af0: 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20  '],.            
0b00: 5b 2f 5e 26 5c 77 2b 3b 28 3f 3a 5c 77 2b 3b 7c  [/^&\w+;(?:\w+;|
0b10: 29 2f 2c 20 27 48 54 4d 4c 45 4e 54 49 54 59 27  )/, 'HTMLENTITY'
0b20: 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20 5b  ],.            [
0b30: 2f 5e 28 3f 3a 6c 7c 64 7c 6e 7c 6d 7c 74 7c 73  /^(?:l|d|n|m|t|s
0b40: 7c 6a 7c 63 7c c3 a7 7c 6c 6f 72 73 71 75 7c 70  |j|c|..|lorsqu|p
0b50: 75 69 73 71 75 7c 6a 75 73 71 75 7c 71 75 6f 69  uisqu|jusqu|quoi
0b60: 71 75 7c 71 75 29 5b 27 e2 80 99 60 5d 2f 69 2c  qu|qu)['...`]/i,
0b70: 20 27 45 4c 50 46 58 27 5d 2c 0a 20 20 20 20 20   'ELPFX'],.     
0b80: 20 20 20 20 20 20 20 5b 2f 5e 5c 64 5c 64 3f 5b         [/^\d\d?[
0b90: 68 6d 5d 5c 64 5c 64 5c 62 2f 2c 20 27 48 4f 55  hm]\d\d\b/, 'HOU
0ba0: 52 27 5d 2c 0a 20 20 20 20 20 20 20 20 20 20 20  R'],.           
0bb0: 20 5b 2f 5e 5c 64 2b 28 3f 3a 65 72 73 3f 7c 6e   [/^\d+(?:ers?|n
0bc0: 64 73 3f 7c 65 73 3f 7c 64 65 73 3f 7c 69 c3 a8  ds?|es?|des?|i..
0bd0: 6d 65 73 3f 7c c3 a8 6d 65 73 3f 7c 65 6d 65 73  mes?|..mes?|emes
0be0: 3f 7c e1 b5 89 ca b3 cb a2 3f 7c e2 81 bf e1 b5  ?|.......?|.....
0bf0: 88 cb a2 3f 7c e1 b5 89 cb a2 3f 7c e1 b5 88 e1  ...?|.....?|....
0c00: b5 89 cb a2 3f 29 5c 62 2f 2c 20 27 4f 52 44 49  ....?)\b/, 'ORDI
0c10: 4e 41 4c 27 5d 2c 0a 20 20 20 20 20 20 20 20 20  NAL'],.         
0c20: 20 20 20 5b 2f 5e 2d 3f 5c 64 2b 28 3f 3a 5b 2e     [/^-?\d+(?:[.
0c30: 2c 5d 5c 64 2b 7c 29 2f 2c 20 27 4e 55 4d 27 5d  ,]\d+|)/, 'NUM']
0c40: 2c 0a 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f  ,.            [/
0c50: 5e 5b 25 e2 80 b0 2b 3d 2a 2f 3c 3e e2 a9 be e2  ^[%...+=*/<>....
0c60: a9 bd 2d 5d 2f 2c 20 27 53 49 47 4e 27 5d 2c 0a  ..-]/, 'SIGN'],.
0c70: 20 20 20 20 20 20 20 20 20 20 20 20 5b 2f 5e 5b              [/^[
0c80: 61 2d 7a 41 2d 5a c3 a0 2d c3 b6 c3 80 2d c3 96  a-zA-Z..-....-..
0c90: 30 2d 39 c3 b8 2d c3 bf c3 98 2d c3 9f c4 80 2d  0-9..-....-....-
0ca0: ca af ef ac 81 2d ef ac 86 5d 2b 28 3f 3a 5b e2  .....-...]+(?:[.
0cb0: 80 99 27 60 2d 5d 5b 61 2d 7a 41 2d 5a c3 a0 2d  ..'`-][a-zA-Z..-
0cc0: c3 b6 c3 80 2d c3 96 30 2d 39 c3 b8 2d c3 bf c3  ....-..0-9..-...
0cd0: 98 2d c3 9f c4 80 2d ca af ef ac 81 2d ef ac 86  .-....-.....-...
0ce0: 5d 2b 29 2a 2f 2c 20 27 57 4f 52 44 27 5d 0a 20  ]+)*/, 'WORD']. 
0cf0: 20 20 20 20 20 20 20 5d 0a 7d 3b 0a 0a 0a 63 6c         ].};...cl
0d00: 61 73 73 20 54 6f 6b 65 6e 69 7a 65 72 20 7b 0a  ass Tokenizer {.
0d10: 0a 20 20 20 20 63 6f 6e 73 74 72 75 63 74 6f 72  .    constructor
0d20: 20 28 73 4c 61 6e 67 29 20 7b 0a 20 20 20 20 20   (sLang) {.     
0d30: 20 20 20 74 68 69 73 2e 73 4c 61 6e 67 20 3d 20     this.sLang = 
0d40: 73 4c 61 6e 67 3b 0a 20 20 20 20 20 20 20 20 69  sLang;.        i
0d50: 66 20 28 21 61 54 6b 7a 50 61 74 74 65 72 6e 73  f (!aTkzPatterns
0d60: 2e 68 61 73 4f 77 6e 50 72 6f 70 65 72 74 79 28  .hasOwnProperty(
0d70: 73 4c 61 6e 67 29 29 20 7b 0a 20 20 20 20 20 20  sLang)) {.      
0d80: 20 20 20 20 20 20 74 68 69 73 2e 73 4c 61 6e 67        this.sLang
0d90: 20 3d 20 22 64 65 66 61 75 6c 74 22 3b 0a 20 20   = "default";.  
0da0: 20 20 20 20 20 20 7d 0a 20 20 20 20 20 20 20 20        }.        
0db0: 74 68 69 73 2e 61 52 75 6c 65 73 20 3d 20 61 54  this.aRules = aT
0dc0: 6b 7a 50 61 74 74 65 72 6e 73 5b 74 68 69 73 2e  kzPatterns[this.
0dd0: 73 4c 61 6e 67 5d 3b 0a 20 20 20 20 7d 0a 0a 20  sLang];.    }.. 
0de0: 20 20 20 2a 20 67 65 6e 54 6f 6b 65 6e 73 20 28     * genTokens (
0df0: 73 54 65 78 74 29 20 7b 0a 20 20 20 20 20 20 20  sText) {.       
0e00: 20 6c 65 74 20 6d 3b 0a 20 20 20 20 20 20 20 20   let m;.        
0e10: 6c 65 74 20 69 4e 65 78 74 20 3d 20 30 3b 0a 20  let iNext = 0;. 
0e20: 20 20 20 20 20 20 20 77 68 69 6c 65 20 28 73 54         while (sT
0e30: 65 78 74 29 20 7b 0a 20 20 20 20 20 20 20 20 20  ext) {.         
0e40: 20 20 20 6c 65 74 20 69 43 75 74 20 3d 20 31 3b     let iCut = 1;
0e50: 0a 20 20 20 20 20 20 20 20 20 20 20 20 6c 65 74  .            let
0e60: 20 69 54 6f 6b 65 6e 20 3d 20 30 3b 0a 20 20 20   iToken = 0;.   
0e70: 20 20 20 20 20 20 20 20 20 66 6f 72 20 28 6c 65           for (le
0e80: 74 20 5b 7a 52 65 67 65 78 2c 20 73 54 79 70 65  t [zRegex, sType
0e90: 5d 20 6f 66 20 74 68 69 73 2e 61 52 75 6c 65 73  ] of this.aRules
0ea0: 29 20 7b 0a 20 20 20 20 20 20 20 20 20 20 20 20  ) {.            
0eb0: 20 20 20 20 74 72 79 20 7b 0a 20 20 20 20 20 20      try {.      
0ec0: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 69 66                if
0ed0: 20 28 28 6d 20 3d 20 7a 52 65 67 65 78 2e 65 78   ((m = zRegex.ex
0ee0: 65 63 28 73 54 65 78 74 29 29 20 21 3d 3d 20 6e  ec(sText)) !== n
0ef0: 75 6c 6c 29 20 7b 0a 20 20 20 20 20 20 20 20 20  ull) {.         
0f00: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 69                 i
0f10: 54 6f 6b 65 6e 20 2b 3d 20 31 3b 0a 20 20 20 20  Token += 1;.    
0f20: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20                  
0f30: 20 20 20 20 79 69 65 6c 64 20 7b 20 22 69 22 3a      yield { "i":
0f40: 20 69 54 6f 6b 65 6e 2c 20 22 73 54 79 70 65 22   iToken, "sType"
0f50: 3a 20 73 54 79 70 65 2c 20 22 73 56 61 6c 75 65  : sType, "sValue
0f60: 22 3a 20 6d 5b 30 5d 2c 20 22 6e 53 74 61 72 74  ": m[0], "nStart
0f70: 22 3a 20 69 4e 65 78 74 2c 20 22 6e 45 6e 64 22  ": iNext, "nEnd"
0f80: 3a 20 69 4e 65 78 74 20 2b 20 6d 5b 30 5d 2e 6c  : iNext + m[0].l
0f90: 65 6e 67 74 68 20 7d 0a 20 20 20 20 20 20 20 20  ength }.        
0fa0: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20                  
0fb0: 69 43 75 74 20 3d 20 6d 5b 30 5d 2e 6c 65 6e 67  iCut = m[0].leng
0fc0: 74 68 3b 0a 20 20 20 20 20 20 20 20 20 20 20 20  th;.            
0fd0: 20 20 20 20 20 20 20 20 20 20 20 20 62 72 65 61              brea
0fe0: 6b 3b 0a 20 20 20 20 20 20 20 20 20 20 20 20 20  k;.             
0ff0: 20 20 20 20 20 20 20 7d 0a 20 20 20 20 20 20 20         }.       
1000: 20 20 20 20 20 20 20 20 20 7d 0a 20 20 20 20 20           }.     
1010: 20 20 20 20 20 20 20 20 20 20 20 63 61 74 63 68             catch
1020: 20 28 65 29 20 7b 0a 20 20 20 20 20 20 20 20 20   (e) {.         
1030: 20 20 20 20 20 20 20 20 20 20 20 68 65 6c 70 65             helpe
1040: 72 73 2e 6c 6f 67 65 72 72 6f 72 28 65 29 3b 0a  rs.logerror(e);.
1050: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20                  
1060: 7d 0a 20 20 20 20 20 20 20 20 20 20 20 20 7d 0a  }.            }.
1070: 20 20 20 20 20 20 20 20 20 20 20 20 69 4e 65 78              iNex
1080: 74 20 2b 3d 20 69 43 75 74 3b 0a 20 20 20 20 20  t += iCut;.     
1090: 20 20 20 20 20 20 20 73 54 65 78 74 20 3d 20 73         sText = s
10a0: 54 65 78 74 2e 73 6c 69 63 65 28 69 43 75 74 29  Text.slice(iCut)
10b0: 3b 0a 20 20 20 20 20 20 20 20 7d 0a 20 20 20 20  ;.        }.    
10c0: 7d 0a 7d 0a 0a 0a 69 66 20 28 74 79 70 65 6f 66  }.}...if (typeof
10d0: 28 65 78 70 6f 72 74 73 29 20 21 3d 3d 20 27 75  (exports) !== 'u
10e0: 6e 64 65 66 69 6e 65 64 27 29 20 7b 0a 20 20 20  ndefined') {.   
10f0: 20 65 78 70 6f 72 74 73 2e 54 6f 6b 65 6e 69 7a   exports.Tokeniz
1100: 65 72 20 3d 20 54 6f 6b 65 6e 69 7a 65 72 3b 0a  er = Tokenizer;.
1110: 7d 0a                                            }.