Grammalecte  Check-in [492169bd6f]

Overview
Comment:[core][js] tokenizer: yield separator one by one
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | core | kill_innerHTML
Files: files | file ages | folders
SHA3-256: 492169bd6ffb8f9e5493fe78da200db1d072d7a7b5e3205e30eee9e239f6686e
User & Date: olr on 2017-07-13 07:08:53
Original Comment: [core] tokenizer: yield separator one by one
Other Links: branch diff | manifest | tags
Context
2017-07-13
07:12
[fr][js] lexicographe: nettoyage check-in: 3996597b39 user: olr tags: fr, kill_innerHTML
07:08
[core][js] tokenizer: yield separator one by one check-in: 492169bd6f user: olr tags: core, kill_innerHTML
07:01
[fr][js] corrige le commit précédent check-in: 622850e787 user: olr tags: fr, kill_innerHTML
Changes

Modified gc_core/js/tokenizer.js from [4eb5311054] to [8720e6c367].

    52     52           let m;
    53     53           let i = 0;
    54     54           while (sText) {
    55     55               let nCut = 1;
    56     56               for (let [zRegex, sType] of this.aRules) {
    57     57                   try {
    58     58                       if ((m = zRegex.exec(sText)) !== null) {
    59         -                        yield { "sType": sType, "sValue": m[0], "nStart": i, "nEnd": i + m[0].length }
           59  +                        if (sType == 'SEPARATOR') {
           60  +                            for (let c of m[0]) {
           61  +                                yield { "sType": sType, "sValue": c, "nStart": i, "nEnd": i + m[0].length }    
           62  +                            }
           63  +                        } else {
           64  +                            yield { "sType": sType, "sValue": m[0], "nStart": i, "nEnd": i + m[0].length }    
           65  +                        }
    60     66                           nCut = m[0].length;
    61     67                           break;
    62     68                       }
    63     69                   }
    64     70                   catch (e) {
    65     71                       helpers.logerror(e);
    66     72                   }