Grammalecte  Check-in [0d1b4ca419]

Overview
Comment:[build][core] analyse tokens outside scope
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | core | build | rg
Files: files | file ages | folders
SHA3-256: 0d1b4ca419a31b1d7d8925058a887d4672e363db31b89bdd4f0f2c1487e60677
User & Date: olr on 2018-06-20 18:32:33
Other Links: branch diff | manifest | tags
Context
2018-06-20
19:17
[fr] avoid regex tokens at the beginning of rules check-in: 93b1465f7d user: olr tags: fr, rg
18:32
[build][core] analyse tokens outside scope check-in: 0d1b4ca419 user: olr tags: build, core, rg
18:30
[fr] conversion: regex rules -> graph rules check-in: 52e8c2eb3a user: olr tags: fr, rg
Changes

Modified compile_rules_graph.py from [5fc68e4b07] to [c0410177dd].

    11     11   dFUNCTIONS = {}
    12     12   
    13     13   
    14     14   def prepareFunction (s, bTokenValue=False):
    15     15       s = s.replace("__also__", "bCondMemo")
    16     16       s = s.replace("__else__", "not bCondMemo")
    17     17       s = re.sub(r"(select|exclude|define)[(][\\](\d+)", 'g_\\1(lToken[\\2+nTokenOffset]', s)
    18         -    s = re.sub(r"(morph|displayInfo)[(]\\(\d+)", 'g_\\1(lToken[\\2+nTokenOffset]', s)
    19         -    s = re.sub(r"(switchGender|has(?:Mas|Fem)Form)[(]\\(\d+)", 'g_\\1(lToken[\\2+nTokenOffset]["sValue"]', s)
    20         -    s = re.sub(r"token\(\s*(\d)", 'nextToken(\\1', s)                                       # token(n)
    21         -    s = re.sub(r"token\(\s*-(\d)", 'prevToken(\\1', s)                                      # token(-n)
           18  +    s = re.sub(r"(morph|analyse|displayInfo)[(]\\(\d+)", 'g_\\1(lToken[\\2+nTokenOffset]', s)
           19  +    s = re.sub(r"(switchGender|has(?:Mas|Fem)Form)[(]\\(\d+)", '\\1(lToken[\\2+nTokenOffset]["sValue"]', s)
           20  +    s = re.sub(r"(morph|analyse)\(>1", 'g_\\1(lToken[nLastToken+1]', s)                     # next token
           21  +    s = re.sub(r"(morph|analyse)\(<1", 'g_\\1(lToken[nTokenOffset]', s)                     # previous token
    22     22       s = re.sub(r"before\(\s*", 'look(s[:m.start()], ', s)                                   # before(s)
    23     23       s = re.sub(r"after\(\s*", 'look(s[m.end():], ', s)                                      # after(s)
    24     24       s = re.sub(r"textarea\(\s*", 'look(s, ', s)                                             # textarea(s)
    25     25       s = re.sub(r"before_chk1\(\s*", 'look_chk1(dDA, s[:m.start()], 0, ', s)                 # before_chk1(s)
    26     26       s = re.sub(r"after_chk1\(\s*", 'look_chk1(dDA, s[m.end():], m.end(), ', s)              # after_chk1(s)
    27     27       s = re.sub(r"textarea_chk1\(\s*", 'look_chk1(dDA, s, 0, ', s)                           # textarea_chk1(s)
    28     28       s = re.sub(r"\bspell *[(]", '_oSpellChecker.isValid(', s)
................................................................................
   331    331   
   332    332       # creating file with all functions callable by rules
   333    333       print("  creating callables...")
   334    334       sPyCallables = "# generated code, do not edit\n"
   335    335       #sJSCallables = "// generated code, do not edit\nconst oEvalFunc = {\n"
   336    336       for sFuncName, sReturn in dFUNCTIONS.items():
   337    337           if sFuncName.startswith("g_c_"): # condition
   338         -            sParams = "lToken, nTokenOffset, sCountry, bCondMemo"
          338  +            sParams = "lToken, nTokenOffset, nLastToken, sCountry, bCondMemo"
   339    339           elif sFuncName.startswith("g_m_"): # message
   340    340               sParams = "lToken, nTokenOffset"
   341    341           elif sFuncName.startswith("g_s_"): # suggestion
   342    342               sParams = "lToken, nTokenOffset"
   343    343           elif sFuncName.startswith("g_p_"): # preprocessor
   344    344               sParams = "lToken"
   345    345           elif sFuncName.startswith("g_d_"): # disambiguator

Modified gc_core/py/lang_core/gc_engine.py from [16c28b42e7] to [5229bc46eb].

   704    704                   try:
   705    705                       if bDebug:
   706    706                           print("ACTION:", sRuleId)
   707    707                           print(dRule[sRuleId])
   708    708                       sOption, sFuncCond, cActionType, sWhat, *eAct = dRule[sRuleId]
   709    709                       # action in lActions: [ condition, action type, replacement/suggestion/action[, iTokenStart, iTokenEnd[, nPriority, message, URL]] ]
   710    710                       if not sOption or dOptions.get(sOption, False):
   711         -                        bCondMemo = not sFuncCond or globals()[sFuncCond](self.lToken, nTokenOffset, sCountry, bCondMemo)
          711  +                        bCondMemo = not sFuncCond or globals()[sFuncCond](self.lToken, nTokenOffset, nLastToken, sCountry, bCondMemo)
   712    712                           if bCondMemo:
   713    713                               if cActionType == "-":
   714    714                                   # grammar error
   715    715                                   nTokenErrorStart = nTokenOffset + eAct[0]
   716    716                                   nTokenErrorEnd = (nTokenOffset + eAct[1])  if eAct[1]  else nLastToken
   717    717                                   nErrorStart = self.nOffsetWithinParagraph + self.lToken[nTokenErrorStart]["nStart"]
   718    718                                   nErrorEnd = self.nOffsetWithinParagraph + self.lToken[nTokenErrorEnd]["nEnd"]