Grammalecte  Check-in [3a97457e25]

Overview
Comment:[build][core] compile rules graph + token sentence checker update
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | core | build | rg
Files: files | file ages | folders
SHA3-256: 3a97457e25092c026d572fcd6507e0a3c7e9c5f9e98f0be273f3d0d854f902cd
User & Date: olr on 2018-05-23 11:48:14
Original Comment: [build][core][bug] compile rules graph: update
Other Links: branch diff | manifest | tags
Context
2018-05-23
12:10
[graphspell][js][bug] remove prefix sign from lemmas check-in: 13109802df user: olr tags: graphspell, rg
11:48
[build][core] compile rules graph + token sentence checker update check-in: 3a97457e25 user: olr tags: build, core, rg
11:47
[core][js][bug] disambigator: always return true check-in: 6e946dad21 user: olr tags: core, rg
Changes

Modified compile_rules_graph.py from [7c9c436423] to [a8ad098a64].

    20     20       s = re.sub(r"isRealStart0 *\(\)", 'before0(["<START>"])', s)
    21     21       s = re.sub(r"isEnd *\(\)", 'after(["<END>", ","])', s)
    22     22       s = re.sub(r"isRealEnd *\(\)", 'after(["<END>"])', s)
    23     23       s = re.sub(r"isEnd0 *\(\)", 'after0(["<END>", ","])', s)
    24     24       s = re.sub(r"isRealEnd0 *\(\)", 'after0(["<END>"])', s)
    25     25       s = re.sub(r"(select|exclude)[(][\\](\d+)", '\\1(lToken[\\2]', s)
    26     26       s = re.sub(r"define[(][\\](\d+)", 'define(lToken[\\1]', s)
    27         -    s = re.sub(r"(morph|morphex|displayInfo)[(][\\](\d+)", '\\1(lToken[\\2])', s)
           27  +    s = re.sub(r"(morph|morphex|displayInfo)[(]\\(\d+)", '\\1(lToken[\\2]', s)
    28     28       s = re.sub(r"token\(\s*(\d)", 'nextToken(\\1', s)                                       # token(n)
    29     29       s = re.sub(r"token\(\s*-(\d)", 'prevToken(\\1', s)                                      # token(-n)
    30     30       s = re.sub(r"before\(\s*", 'look(s[:m.start()], ', s)                                   # before(s)
    31     31       s = re.sub(r"after\(\s*", 'look(s[m.end():], ', s)                                      # after(s)
    32     32       s = re.sub(r"textarea\(\s*", 'look(s, ', s)                                             # textarea(s)
    33     33       s = re.sub(r"before_chk1\(\s*", 'look_chk1(dDA, s[:m.start()], 0, ', s)                 # before_chk1(s)
    34     34       s = re.sub(r"after_chk1\(\s*", 'look_chk1(dDA, s[m.end():], m.end(), ', s)              # after_chk1(s)
................................................................................
   111    111               mURL = re.search("[|] *(https?://.*)", sMsg)
   112    112               if mURL:
   113    113                   sURL = mURL.group(1).strip()
   114    114                   sMsg = sMsg[:mURL.start(0)].strip()
   115    115               if sMsg[0:1] == "=":
   116    116                   sMsg = prepareFunction(sMsg[1:])
   117    117                   lFUNCTIONS.append(("g_m_"+sIdAction, sMsg))
   118         -                for x in re.finditer("group[(](\d+)[)]", sMsg):
          118  +                for x in re.finditer("group[(](\\d+)[)]", sMsg):
   119    119                       if int(x.group(1)) > nGroup:
   120    120                           print("# Error in groups in message at line " + sIdAction + " ("+str(nGroup)+" groups only)")
   121    121                   sMsg = "=g_m_"+sIdAction
   122    122               else:
   123    123                   for x in re.finditer(r"\\(\d+)", sMsg):
   124    124                       if int(x.group(1)) > nGroup:
   125    125                           print("# Error in groups in message at line " + sIdAction + " ("+str(nGroup)+" groups only)")
................................................................................
   126    126                   if re.search("[.]\\w+[(]", sMsg):
   127    127                       print("# Error in message at line " + sIdAction + ":  This message looks like code. Line should begin with =")
   128    128               
   129    129       if sAction[0:1] == "=" or cAction == "=":
   130    130           if "define" in sAction and not re.search(r"define\(\\\d+ *, *\[.*\] *\)", sAction):
   131    131               print("# Error in action at line " + sIdAction + ": second argument for define must be a list of strings")
   132    132           sAction = prepareFunction(sAction)
   133         -        for x in re.finditer("group[(](\d+)[)]", sAction):
          133  +        for x in re.finditer("group[(](\\d+)[)]", sAction):
   134    134               if int(x.group(1)) > nGroup:
   135    135                   print("# Error in groups in replacement at line " + sIdAction + " ("+str(nGroup)+" groups only)")
   136    136       else:
   137    137           for x in re.finditer(r"\\(\d+)", sAction):
   138    138               if int(x.group(1)) > nGroup:
   139    139                   print("# Error in groups in replacement at line " + sIdAction + " ("+str(nGroup)+" groups only)")
   140    140           if re.search("[.]\\w+[(]|sugg\\w+[(]", sAction):
................................................................................
   265    265       # Graph creation
   266    266       for e in lPreparedRule:
   267    267           print(e)
   268    268   
   269    269       oDARG = darg.DARG(lPreparedRule, sLang)
   270    270       oRuleGraph = oDARG.createGraph()
   271    271   
          272  +    # creating file with all functions callable by rules
          273  +    print("  creating callables...")
          274  +    sPyCallables = "# generated code, do not edit\n"
          275  +    #sJSCallables = "// generated code, do not edit\nconst oEvalFunc = {\n"
          276  +    for sFuncName, sReturn in lFUNCTIONS:
          277  +        if sFuncName.startswith("g_c_"): # condition
          278  +            sParams = "lToken, sCountry, bCondMemo"
          279  +        elif sFuncName.startswith("g_m_"): # message
          280  +            sParams = "lToken"
          281  +        elif sFuncName.startswith("g_s_"): # suggestion
          282  +            sParams = "lToken"
          283  +        elif sFuncName.startswith("g_p_"): # preprocessor
          284  +            sParams = "lToken"
          285  +        elif sFuncName.startswith("g_d_"): # disambiguator
          286  +            sParams = "lToken"
          287  +        else:
          288  +            print("# Unknown function type in [" + sFuncName + "]")
          289  +            continue
          290  +        sPyCallables += "def {} ({}):\n".format(sFuncName, sParams)
          291  +        sPyCallables += "    return " + sReturn + "\n"
          292  +        #sJSCallables += "    {}: function ({})".format(sFuncName, sParams) + " {\n"
          293  +        #sJSCallables += "        return " + jsconv.py2js(sReturn) + ";\n"
          294  +        #sJSCallables += "    },\n"
          295  +    #sJSCallables += "}\n"
          296  +
   272    297       # Result
   273    298       d = {
   274         -        "graph_callables": None,
          299  +        "graph_callables": sPyCallables,
   275    300           "graph_gctests": None,
   276    301           "rules_graph": oRuleGraph,
   277    302           "rules_actions": dACTIONS
   278    303       }
   279    304   
   280    305       return d
   281    306   
   282    307   

Modified gc_core/py/lang_core/gc_engine.py from [29b43c054f] to [070a603bc6].

    74     74       for iStart, iEnd in _getSentenceBoundaries(sText):
    75     75           if 4 < (iEnd - iStart) < 2000:
    76     76               dDA.clear()
    77     77               try:
    78     78                   # regex parser
    79     79                   _, errs = _proofread(sText[iStart:iEnd], sRealText[iStart:iEnd], iStart, False, dDA, dPriority, sCountry, dOpt, bDebug, bContext)
    80     80                   aErrors.update(errs)
           81  +                # token parser
           82  +                oSentence = TokenSentence(sText[iStart:iEnd], sRealText[iStart:iEnd], iStart, dPriority, sCountry, dOpt, bDebug, bContext)
           83  +                oSentence.parse()
    81     84               except:
    82     85                   raise
    83     86       return aErrors.values() # this is a view (iterable)
    84     87   
    85     88   
    86     89   def _getSentenceBoundaries (sText):
    87     90       iStart = _zBeginOfParagraph.match(sText).end()
................................................................................
   564    567   
   565    568   ${plugins}
   566    569   
   567    570   
   568    571   #### CALLABLES (generated code)
   569    572   
   570    573   ${callables}
          574  +
          575  +
          576  +
          577  +#### TOKEN SENTENCE CHECKER
          578  +
          579  +class TokenSentence:
          580  +
          581  +    def __init__ (self, sSentence, sSentence0, iStart, dPriority, sCountry, dOpt, bDebug, bContext):
          582  +        self.sSentence = sSentence
          583  +        self.sSentence0 = sSentence0
          584  +        self.iStart = iStart
          585  +        self.lToken = list(_oTokenizer.genTokens(sSentence))
          586  +
          587  +    def parse (self):
          588  +        dErr = {}
          589  +        lPointer = []
          590  +        for dToken in self.lToken:
          591  +            for i, dPointer in enumerate(lPointer):
          592  +                bValid = False
          593  +                for dNode in self._getNextMatchingNodes(dToken, dPointer["dNode"]):
          594  +                    dPointer["nOffset"] = dToken["i"]
          595  +                    dPointer["dNode"] = dNode
          596  +                    bValid = True
          597  +                if not bValid:
          598  +                    del lPointer[i]
          599  +            for dNode in self._getNextMatchingNodes(dToken, dGraph):
          600  +                lPointer.append({"nOffset": 0, "dNode": dNode})
          601  +            for dPointer in lPointer:
          602  +                if "<rules>" in dPointer["dNode"]:
          603  +                    for dNode in dGraph[dPointer["dNode"]["<rules>"]]:
          604  +                        dErr = self._executeActions(dNode, nOffset)
          605  +        return dErr
          606  +
          607  +    def _getNextMatchingNodes (self, dToken, dNode):
          608  +        # token value
          609  +        if dToken["sValue"] in dNode:
          610  +            yield dGraph[dNode[dToken["sValue"]]]
          611  +        # token lemmas
          612  +        for sLemma in _oSpellChecker.getLemma(dToken["sValue"]):
          613  +            if sLemma in dNode:
          614  +                yield dGraph[dNode[sLemma]]
          615  +        # universal arc
          616  +        if "*" in dNode:
          617  +            yield dGraph[dNode["*"]]
          618  +        # regex arcs
          619  +        if "~" in dNode:
          620  +            for sRegex in dNode["~"]:
          621  +                for sMorph in _oSpellChecker.getMorph(dToken["sValue"]):
          622  +                    if re.search(sRegex, sMorph):
          623  +                        yield dGraph[dNode["~"][sRegex]]
          624  +
          625  +    def _executeActions (self, dNode, nOffset):
          626  +        for sLineId, nextNodeKey in dNode.items():
          627  +            for sArc in dGraph[nextNodeKey]:
          628  +                print(sArc)
          629  +                bCondMemo = None
          630  +                sFuncCond, cActionType, sWhat, *eAct = dRule[sArc]
          631  +                # action in lActions: [ condition, action type, replacement/suggestion/action[, iGroupStart, iGroupEnd[, message, URL]] ]
          632  +                try:
          633  +                    bCondMemo = not sFuncCond or globals()[sFuncCond](self, sCountry, bCondMemo)
          634  +                    if bCondMemo:
          635  +                        if cActionType == "-":
          636  +                            # grammar error
          637  +                            print("-")
          638  +                            nErrorStart = nSentenceOffset + m.start(eAct[0])
          639  +                            nErrorEnd = nSentenceOffset + m.start(eAct[1])
          640  +                            if nErrorStart not in dErrs or nPriority > dPriority[nErrorStart]:
          641  +                                dErrs[nErrorStart] = _createError(self, sWhat, nErrorStart, nErrorEnd, sLineId, bUppercase, eAct[2], eAct[3], bIdRule, sOption, bContext)
          642  +                                dPriority[nErrorStart] = nPriority
          643  +                        elif cActionType == "~":
          644  +                            # text processor
          645  +                            print("~")
          646  +                            self._rewrite(sWhat, nErrorStart, nErrorEnd)
          647  +                        elif cActionType == "@":
          648  +                            # jump
          649  +                            print("@")
          650  +                            self._jump(sWhat)
          651  +                        elif cActionType == "=":
          652  +                            # disambiguation
          653  +                            print("=")
          654  +                            globals()[sWhat](self.lToken)
          655  +                        elif cActionType == ">":
          656  +                            # we do nothing, this test is just a condition to apply all following actions
          657  +                            print(">")
          658  +                            pass
          659  +                        else:
          660  +                            print("# error: unknown action at " + sLineId)
          661  +                    elif cActionType == ">":
          662  +                        break
          663  +                except Exception as e:
          664  +                    raise Exception(str(e), "# " + sLineId + " # " + sRuleId)
          665  +
          666  +    def _createWriterError (self):
          667  +        d = {}
          668  +        return d
          669  +
          670  +    def _createDictError (self):
          671  +        d = {}
          672  +        return d
          673  +
          674  +    def _rewrite (self, sWhat, nErrorStart, nErrorEnd):
          675  +        "text processor: rewrite tokens between <nErrorStart> and <nErrorEnd> position"
          676  +        lTokenValue = sWhat.split("|")
          677  +        if len(lTokenValue) != (nErrorEnd - nErrorStart + 1):
          678  +            print("Error. Text processor: number of replacements != number of tokens.")
          679  +            return
          680  +        for i, sValue in zip(range(nErrorStart, nErrorEnd+1), lTokenValue):
          681  +            self.lToken[i]["sValue"] = sValue
          682  +
          683  +    def _jump (self, sWhat):
          684  +        try:
          685  +            nFrom, nTo = sWhat.split(">")
          686  +            self.lToken[int(nFrom)]["iJump"] = int(nTo)
          687  +        except:
          688  +            print("# Error. Jump failed: ", sWhat)
          689  +            traceback.print_exc()
          690  +            return
          691  +
          692  +
          693  +#### Analyse tokens
          694  +
          695  +def g_morph (dToken, sPattern, bStrict=True):
          696  +    "analyse a token, return True if <sPattern> in morphologies"
          697  +    if "lMorph" in dToken:
          698  +        lMorph = dToken["lMorph"]
          699  +    else:
          700  +        lMorph = _oSpellChecker.getMorph(dToken["sValue"])
          701  +        if not lMorph:
          702  +            return False
          703  +    zPattern = re.compile(sPattern)
          704  +    if bStrict:
          705  +        return all(zPattern.search(sMorph)  for sMorph in lMorph)
          706  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
          707  +
          708  +def g_morphex (dToken, sPattern, sNegPattern):
          709  +    "analyse a token, return True if <sNegPattern> not in morphologies and <sPattern> in morphologies"
          710  +    if "lMorph" in dToken:
          711  +        lMorph = dToken["lMorph"]
          712  +    else:
          713  +        lMorph = _oSpellChecker.getMorph(dToken["sValue"])
          714  +        if not lMorph:
          715  +            return False
          716  +    # check negative condition
          717  +    zNegPattern = re.compile(sNegPattern)
          718  +    if any(zNegPattern.search(sMorph)  for sMorph in lMorph):
          719  +        return False
          720  +    # search sPattern
          721  +    zPattern = re.compile(sPattern)
          722  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
          723  +
          724  +def g_analyse (dToken, sPattern, bStrict=True):
          725  +    "analyse a token, return True if <sPattern> in morphologies (disambiguation off)"
          726  +    lMorph = _oSpellChecker.getMorph(dToken["sValue"])
          727  +    if not lMorph:
          728  +        return False
          729  +    zPattern = re.compile(sPattern)
          730  +    if bStrict:
          731  +        return all(zPattern.search(sMorph)  for sMorph in lMorph)
          732  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
          733  +
          734  +
          735  +def g_analysex (dToken, sPattern, sNegPattern):
          736  +    "analyse a token, return True if <sNegPattern> not in morphologies and <sPattern> in morphologies (disambiguation off)"
          737  +    lMorph = _oSpellChecker.getMorph(dToken["sValue"])
          738  +    if not lMorph:
          739  +        return False
          740  +    # check negative condition
          741  +    zNegPattern = re.compile(sNegPattern)
          742  +    if any(zNegPattern.search(sMorph)  for sMorph in lMorph):
          743  +        return False
          744  +    # search sPattern
          745  +    zPattern = re.compile(sPattern)
          746  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
          747  +
          748  +
          749  +#### Go outside the rule scope
          750  +
          751  +def g_nextToken (i):
          752  +    pass
          753  +
          754  +def g_prevToken (i):
          755  +    pass
          756  +
          757  +def g_look ():
          758  +    pass
          759  +
          760  +def g_lookAndCheck ():
          761  +    pass
          762  +
          763  +
          764  +#### Disambiguator
          765  +
          766  +def g_select (dToken, sPattern, lDefault=None):
          767  +    "select morphologies for <dToken> according to <sPattern>, always return True"
          768  +    lMorph = dToken["lMorph"]  if "lMorph" in dToken  else _oSpellChecker.getMorph(dToken["sValue"])
          769  +    if not lMorph or len(lMorph) == 1:
          770  +        return True
          771  +    lSelect = [ sMorph  for sMorph in lMorph  if re.search(sPattern, sMorph) ]
          772  +    if lSelect:
          773  +        if len(lSelect) != len(lMorph):
          774  +            dToken["lMorph"] = lSelect
          775  +    elif lDefault:
          776  +        dToken["lMorph"] = lDefault
          777  +    return True
          778  +
          779  +
          780  +def g_exclude (dToken, sPattern, lDefault=None):
          781  +    "select morphologies for <dToken> according to <sPattern>, always return True"
          782  +    lMorph = dToken["lMorph"]  if "lMorph" in dToken  else _oSpellChecker.getMorph(dToken["sValue"])
          783  +    if not lMorph or len(lMorph) == 1:
          784  +        return True
          785  +    lSelect = [ sMorph  for sMorph in lMorph  if not re.search(sPattern, sMorph) ]
          786  +    if lSelect:
          787  +        if len(lSelect) != len(lMorph):
          788  +            dToken["lMorph"] = lSelect
          789  +    elif lDefault:
          790  +        dToken["lMorph"] = lDefault
          791  +    return True
          792  +
          793  +
          794  +def g_define (dToken, lMorph):
          795  +    "set morphologies of <dToken>, always return True"
          796  +    dToken["lMorph"] = lMorph
          797  +    return True
          798  +
          799  +
          800  +#### CALLABLES (generated code)
          801  +
          802  +${graph_callables}

Modified gc_core/py/lang_core/gc_sentence.py from [90cbca3aed] to [c68dc1622f].

     1      1   # Sentence checker
     2      2   
     3      3   from ..graphspell.tokenizer import Tokenizer
     4         -from .gc_graph import dGraph
            4  +from .gc_rules_graph import dGraph
     5      5   
     6      6   
     7      7   oTokenizer = Tokenizer("${lang}")
     8      8   
     9      9   
    10         -class Sentence:
           10  +class TokenSentence:
    11     11   
    12     12       def __init__ (self, sSentence, sSentence0, nOffset):
    13     13           self.sSentence = sSentence
    14     14           self.sSentence0 = sSentence0
    15     15           self.nOffset = nOffset
    16     16           self.lToken = list(oTokenizer.genTokens())
    17     17   
................................................................................
    28     28                   if not bValid:
    29     29                       del lPointer[i]
    30     30               for dNode in self._getNextMatchingNodes(dToken, dGraph):
    31     31                   lPointer.append({"nOffset": 0, "dNode": dNode})
    32     32               for dPointer in lPointer:
    33     33                   if "<rules>" in dPointer["dNode"]:
    34     34                       for dNode in dGraph[dPointer["dNode"]["<rules>"]]:
    35         -                        dErr = self._executeActions(dNode)
           35  +                        dErr = self._executeActions(dNode, nOffset)
    36     36           return dErr
    37     37   
    38     38       def _getNextMatchingNodes (self, dToken, dNode):
           39  +        # token value
    39     40           if dToken["sValue"] in dNode:
    40     41               yield dGraph[dNode[dToken["sValue"]]]
    41         -        for sLemma in dToken["sLemma"]:
           42  +        # token lemmas
           43  +        for sLemma in dToken["lLemma"]:
    42     44               if sLemma in dNode:
    43         -                yield dGraph[dNode[dToken["sValue"]]]
           45  +                yield dGraph[dNode[sLemma]]
           46  +        # universal arc
           47  +        if "*" in dNode:
           48  +            yield dGraph[dNode["*"]]
           49  +        # regex arcs
    44     50           if "~" in dNode:
    45     51               for sRegex in dNode["~"]:
    46     52                   for sMorph in dToken["lMorph"]:
    47     53                       if re.search(sRegex, sMorph):
    48     54                           yield dGraph[dNode["~"][sRegex]]
    49     55   
    50         -    def _executeActions (self, dNode):
           56  +    def _executeActions (self, dNode, nOffset):
    51     57           for sLineId, nextNodeKey in dNode.items():
    52     58               for sArc in dGraph[nextNodeKey]:
    53     59                   bCondMemo = None
    54     60                   sFuncCond, cActionType, sWhat, *eAct = dRule[sArc]
    55     61                   # action in lActions: [ condition, action type, replacement/suggestion/action[, iGroupStart, iGroupEnd[, message, URL]] ]
    56     62                   try:
    57         -                    bCondMemo = not sFuncCond or globals()[sFuncCond](self, dDA, sCountry, bCondMemo)
           63  +                    bCondMemo = not sFuncCond or globals()[sFuncCond](self, sCountry, bCondMemo)
    58     64                       if bCondMemo:
    59     65                           if cActionType == "-":
    60     66                               # grammar error
    61     67                               nErrorStart = nSentenceOffset + m.start(eAct[0])
    62     68                               nErrorEnd = nSentenceOffset + m.start(eAct[1])
    63     69                               if nErrorStart not in dErrs or nPriority > dPriority[nErrorStart]:
    64     70                                   dErrs[nErrorStart] = _createError(self, sWhat, nErrorStart, nErrorEnd, sLineId, bUppercase, eAct[2], eAct[3], bIdRule, sOption, bContext)
    65     71                                   dPriority[nErrorStart] = nPriority
    66     72                           elif cActionType == "~":
    67     73                               # text processor
    68         -                            self.lToken = _rewrite(self, sWhat, nErrorStart, nErrorEnd, bUppercase)
    69         -                            bChange = True
           74  +                            self._rewrite(sWhat, nErrorStart, nErrorEnd)
    70     75                           elif cActionType == "@":
    71         -                            # text processor
    72         -                            self.lToken = _rewrite(self, sWhat, nErrorStart, nErrorEnd, bUppercase)
    73         -                            bChange = True
           76  +                            # jump
           77  +                            self._jump(sWhat)
    74     78                           elif cActionType == "=":
    75     79                               # disambiguation
    76         -                            globals()[sWhat](self, dDA)
           80  +                            globals()[sWhat](self.lToken)
    77     81                           elif cActionType == ">":
    78     82                               # we do nothing, this test is just a condition to apply all following actions
    79     83                               pass
    80     84                           else:
    81         -                            echo("# error: unknown action at " + sLineId)
           85  +                            print("# error: unknown action at " + sLineId)
    82     86                       elif cActionType == ">":
    83     87                           break
    84     88                   except Exception as e:
    85     89                       raise Exception(str(e), "# " + sLineId + " # " + sRuleId)
    86     90   
    87     91       def _createWriterError (self):
    88     92           d = {}
    89     93           return d
    90     94   
    91     95       def _createDictError (self):
    92     96           d = {}
    93     97           return d
    94     98   
           99  +    def _rewrite (self, sWhat, nErrorStart, nErrorEnd):
          100  +        "text processor: rewrite tokens between <nErrorStart> and <nErrorEnd> position"
          101  +        lTokenValue = sWhat.split("|")
          102  +        if len(lTokenValue) != (nErrorEnd - nErrorStart + 1):
          103  +            print("Error. Text processor: number of replacements != number of tokens.")
          104  +            return
          105  +        for i, sValue in zip(range(nErrorStart, nErrorEnd+1), lTokenValue):
          106  +            self.lToken[i]["sValue"] = sValue
    95    107   
    96         -#### Common functions
    97         -
    98         -def option ():
    99         -    pass
          108  +    def _jump (self, sWhat):
          109  +        try:
          110  +            nFrom, nTo = sWhat.split(">")
          111  +            self.lToken[int(nFrom)]["iJump"] = int(nTo)
          112  +        except:
          113  +            print("# Error. Jump failed: ", sWhat)
          114  +            traceback.print_exc()
          115  +            return
   100    116   
   101    117   
   102    118   #### Analyse tokens
   103    119   
   104         -def morph ():
   105         -    pass
          120  +def g_morph (dToken, sPattern, bStrict=True):
          121  +    "analyse a token, return True if <sPattern> in morphologies"
          122  +    if "lMorph" in dToken:
          123  +        lMorph = dToken["lMorph"]
          124  +    else:
          125  +        if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          126  +            return False
          127  +        if not _dAnalyses[dToken["sValue"]]:
          128  +            return False
          129  +        lMorph = _dAnalyses[dToken["sValue"]]
          130  +    zPattern = re.compile(sPattern)
          131  +    if bStrict:
          132  +        return all(zPattern.search(sMorph)  for sMorph in lMorph)
          133  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
   106    134   
   107         -def morphex ():
          135  +def g_morphex (dToken, sPattern, sNegPattern):
          136  +    "analyse a token, return True if <sNegPattern> not in morphologies and <sPattern> in morphologies"
          137  +    if "lMorph" in dToken:
          138  +        lMorph = dToken["lMorph"]
          139  +    else:
          140  +        if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          141  +            return False
          142  +        if not _dAnalyses[dToken["sValue"]]:
          143  +            return False
          144  +        lMorph = _dAnalyses[dToken["sValue"]]
          145  +    # check negative condition
          146  +    zNegPattern = re.compile(sNegPattern)
          147  +    if any(zNegPattern.search(sMorph)  for sMorph in lMorph):
          148  +        return False
          149  +    # search sPattern
          150  +    zPattern = re.compile(sPattern)
          151  +    return any(zPattern.search(sMorph)  for sMorph in lMorph)
          152  +
          153  +def g_analyse (dToken, sPattern, bStrict=True):
          154  +    "analyse a token, return True if <sPattern> in morphologies (disambiguation off)"
          155  +    if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          156  +        return False
          157  +    if not _dAnalyses[dToken["sValue"]]:
          158  +        return False
          159  +    zPattern = re.compile(sPattern)
          160  +    if bStrict:
          161  +        return all(zPattern.search(sMorph)  for sMorph in _dAnalyses[dToken["sValue"]])
          162  +    return any(zPattern.search(sMorph)  for sMorph in _dAnalyses[dToken["sValue"]])
          163  +
          164  +
          165  +def g_analysex (dToken, sPattern, sNegPattern):
          166  +    "analyse a token, return True if <sNegPattern> not in morphologies and <sPattern> in morphologies (disambiguation off)"
          167  +    if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          168  +        return False
          169  +    if not _dAnalyses[dToken["sValue"]]:
          170  +        return False
          171  +    # check negative condition
          172  +    zNegPattern = re.compile(sNegPattern)
          173  +    if any(zNegPattern.search(sMorph)  for sMorph in _dAnalyses[dToken["sValue"]]):
          174  +        return False
          175  +    # search sPattern
          176  +    zPattern = re.compile(sPattern)
          177  +    return any(zPattern.search(sMorph)  for sMorph in _dAnalyses[dToken["sValue"]])
          178  +
          179  +
          180  +#### Go outside the rule scope
          181  +
          182  +def g_nextToken (i):
   108    183       pass
   109    184   
   110         -def analyse ():
          185  +def g_prevToken (i):
   111    186       pass
   112    187   
   113         -def analysex ():
   114         -    pass
   115         -
   116         -
   117         -#### Go outside scope
   118         -
   119         -def nextToken ():
          188  +def g_look ():
   120    189       pass
   121    190   
   122         -def prevToken ():
   123         -    pass
   124         -
   125         -def look ():
   126         -    pass
   127         -
   128         -def lookAndCheck ():
          191  +def g_lookAndCheck ():
   129    192       pass
   130    193   
   131    194   
   132    195   #### Disambiguator
   133    196   
   134         -def select ():
   135         -    pass
          197  +def g_select (dToken, sPattern, lDefault=None):
          198  +    "select morphologies for <dToken> according to <sPattern>, always return True"
          199  +    if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          200  +        return True
          201  +    if len(_dAnalyses[dToken["sValue"]]) == 1:
          202  +        return True
          203  +    lMorph = dToken["lMorph"] or _dAnalyses[dToken["sValue"]]
          204  +    lSelect = [ sMorph  for sMorph in lMorph  if re.search(sPattern, sMorph) ]
          205  +    if lSelect:
          206  +        if len(lSelect) != len(lMorph):
          207  +            dToken["lMorph"] = lSelect
          208  +    elif lDefault:
          209  +        dToken["lMorph"] = lDefault
          210  +    return True
          211  +
          212  +
          213  +def g_exclude (dToken, sPattern, lDefault=None):
          214  +    "select morphologies for <dToken> according to <sPattern>, always return True"
          215  +    if dToken["sValue"] not in _dAnalyses and not _storeMorphFromFSA(dToken["sValue"]):
          216  +        return True
          217  +    if len(_dAnalyses[dToken["sValue"]]) == 1:
          218  +        return True
          219  +    lMorph = dToken["lMorph"] or _dAnalyses[dToken["sValue"]]
          220  +    lSelect = [ sMorph  for sMorph in lMorph  if not re.search(sPattern, sMorph) ]
          221  +    if lSelect:
          222  +        if len(lSelect) != len(lMorph):
          223  +            dToken["lMorph"] = lSelect
          224  +    elif lDefault:
          225  +        dToken["lMorph"] = lDefault
          226  +    return True
          227  +
          228  +
          229  +def g_define (dToken, lMorph):
          230  +    "set morphologies of <dToken>, always return True"
          231  +    dToken["lMorph"] = lMorph
          232  +    return True
          233  +
   136    234   
   137         -def exclude ():
   138         -    pass
          235  +#### CALLABLES (generated code)
   139    236   
   140         -def define ():
   141         -    pass
          237  +${graph_callables}