Grammalecte  Check-in [b5ee5c589f]

Overview
Comment:[build] check token validity
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | build
Files: files | file ages | folders
SHA3-256: b5ee5c589f7db37389be4f363c770ea4eb848e127f605f60312246203cfa7941
User & Date: olr on 2020-11-23 10:32:13
Other Links: manifest | tags
Context
2020-11-23
20:28
[fr] corrections des messages et ajustements check-in: ab708276ae user: olr tags: fr, trunk
10:32
[build] check token validity check-in: b5ee5c589f user: olr tags: build, trunk
2020-11-21
21:42
[core][fr] tests update check-in: 6639af3543 user: olr tags: core, fr, trunk
Changes

Modified compile_rules_graph.py from [38bc0eb9da] to [dfe32ac11e].

   103    103           self.dOptPriority = dOptPriority
   104    104           self.dAntiPatterns = {}
   105    105           self.dActions = {}
   106    106           self.dFuncName = {}
   107    107           self.dFunctions = {}
   108    108           self.dLemmas = {}
   109    109   
          110  +    def createGraphAndActions (self, lRuleLine):
          111  +        "create a graph as a dictionary with <lRuleLine>"
          112  +        fStartTimer = time.time()
          113  +        print("{:>8,} rules in {:<30} ".format(len(lRuleLine), f"<{self.sGraphName}|{self.sGraphCode}>"), end="")
          114  +        lPreparedRule = []
          115  +        for i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority in lRuleLine:
          116  +            for aRule in self.createRule(i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority):
          117  +                lPreparedRule.append(aRule)
          118  +        # Debugging
          119  +        if False:
          120  +            print("\nRULES:")
          121  +            for e in lPreparedRule:
          122  +                if e[-2] == "##2211":
          123  +                    print(e)
          124  +        # Graph creation
          125  +        oDARG = darg.DARG(lPreparedRule, self.sLang)
          126  +        dGraph = oDARG.createGraph()
          127  +        print(oDARG, end="")
          128  +        # debugging
          129  +        if False:
          130  +            print("\nGRAPH:", self.sGraphName)
          131  +            for k, v in dGraph.items():
          132  +                print(k, "\t", v)
          133  +        print("\tin {:>8.2f} s".format(time.time()-fStartTimer))
          134  +        sPyCallables, sJSCallables = self.createCallables()
          135  +        return dGraph, self.dActions, sPyCallables, sJSCallables, self.dLemmas
          136  +
   110    137       def _genTokenLines (self, sTokenLine):
   111    138           "tokenize a string and return a list of lines of tokens"
   112    139           lTokenLines = []
   113         -        for sTokBlock in sTokenLine.split():
          140  +        nFirstNullable = 0
          141  +        nLastNullable = 0
          142  +        for n, sTokBlock in enumerate(sTokenLine.split(), 1):
   114    143               # replace merger characters by spaces
   115    144               if "␣" in sTokBlock:
   116    145                   sTokBlock = sTokBlock.replace("␣", " ")
   117    146               # optional token?
   118    147               bNullPossible = sTokBlock.startswith("?") and sTokBlock.endswith("¿")
   119    148               if bNullPossible:
   120    149                   sTokBlock = sTokBlock[1:-1]
          150  +                if nFirstNullable == 0:
          151  +                    nFirstNullable = n
          152  +                nLastNullable = n
   121    153               # token with definition?
          154  +            if sTokBlock.startswith("(") and sTokBlock.endswith(")"):
          155  +                nFirstNullable = -1
   122    156               if sTokBlock.startswith("({") and sTokBlock.endswith("})") and sTokBlock[1:-1] in self.dDef:
   123    157                   sTokBlock = "(" + self.dDef[sTokBlock[1:-1]] + ")"
   124    158               elif sTokBlock.startswith("{") and sTokBlock.endswith("}") and sTokBlock in self.dDef:
   125    159                   sTokBlock = self.dDef[sTokBlock]
   126    160               if ( (sTokBlock.startswith("[") and sTokBlock.endswith("]")) or (sTokBlock.startswith("([") and sTokBlock.endswith("])")) ):
   127    161                   # multiple token
   128    162                   bSelectedGroup = sTokBlock.startswith("(") and sTokBlock.endswith(")")
................................................................................
   161    195                               lNew = list(aRule)
   162    196                               lNew.append(sTokBlock)
   163    197                               lNewTemp.append(lNew)
   164    198                           lTokenLines.extend(lNewTemp)
   165    199                       else:
   166    200                           for aRule in lTokenLines:
   167    201                               aRule.append(sTokBlock)
          202  +        nLastNullable = nLastNullable - n - 1
   168    203           for aRule in lTokenLines:
   169         -            yield aRule
          204  +            yield aRule, nFirstNullable, nLastNullable
   170    205   
   171    206       def _createTokenList (self, sTokBlock):
   172    207           "return a list of tokens from a block of tokens"
   173    208           lToken = []
   174    209           for sToken in sTokBlock[1:-1].split("|"):
   175    210               if "+" in sToken and not sToken.startswith("+"):
   176    211                   for sCode in self.dDecl:
................................................................................
   180    215                           for sSuffix in self.dDecl[sCode]:
   181    216                               lToken.append(sToken+sSuffix)
   182    217                           break
   183    218               else:
   184    219                   lToken.append(sToken)
   185    220           return lToken
   186    221   
   187         -    def createGraphAndActions (self, lRuleLine):
   188         -        "create a graph as a dictionary with <lRuleLine>"
   189         -        fStartTimer = time.time()
   190         -        print("{:>8,} rules in {:<30} ".format(len(lRuleLine), f"<{self.sGraphName}|{self.sGraphCode}>"), end="")
   191         -        lPreparedRule = []
   192         -        for i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority in lRuleLine:
   193         -            for aRule in self.createRule(i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority):
   194         -                lPreparedRule.append(aRule)
   195         -        # Debugging
   196         -        if False:
   197         -            print("\nRULES:")
   198         -            for e in lPreparedRule:
   199         -                if e[-2] == "##2211":
   200         -                    print(e)
   201         -        # Graph creation
   202         -        oDARG = darg.DARG(lPreparedRule, self.sLang)
   203         -        dGraph = oDARG.createGraph()
   204         -        print(oDARG, end="")
   205         -        # debugging
   206         -        if False:
   207         -            print("\nGRAPH:", self.sGraphName)
   208         -            for k, v in dGraph.items():
   209         -                print(k, "\t", v)
   210         -        print("\tin {:>8.2f} s".format(time.time()-fStartTimer))
   211         -        sPyCallables, sJSCallables = self.createCallables()
   212         -        return dGraph, self.dActions, sPyCallables, sJSCallables, self.dLemmas
   213         -
   214    222       def createRule (self, iLine, sRuleName, sTokenLine, iActionBlock, lActions, nPriority):
   215    223           "generator: create rule as list"
   216    224           # print(iLine, "//", sRuleName, "//", sTokenLine, "//", lActions, "//", nPriority)
   217    225           if sTokenLine.startswith("!!") and sTokenLine.endswith("¡¡"):
   218    226               # antipattern
   219    227               sTokenLine = sTokenLine[2:-2].strip()
   220    228               if sRuleName not in self.dAntiPatterns:
   221    229                   self.dAntiPatterns[sRuleName]= []
   222         -            for lToken in self._genTokenLines(sTokenLine):
          230  +            for lToken, _, _ in self._genTokenLines(sTokenLine):
   223    231                   self.dAntiPatterns[sRuleName].append(lToken)
   224    232           else:
   225    233               # pattern
   226         -            for lToken in self._genTokenLines(sTokenLine):
          234  +            for lToken, nFirstNullable, nLastNullable in self._genTokenLines(sTokenLine):
   227    235                   if sRuleName in self.dAntiPatterns and lToken in self.dAntiPatterns[sRuleName]:
   228    236                       # <lToken> matches an antipattern -> discard
   229    237                       continue
   230    238                   # Calculate positions
   231    239                   dPos = {}   # key: iGroup, value: iToken
   232    240                   iGroup = 0
   233    241                   #if iLine == 15818: # debug
................................................................................
   241    249                       if sToken.startswith(">") and sToken != ">" and sToken[1:] not in self.dLemmas:
   242    250                           self.dLemmas[sToken[1:]] = iLine
   243    251                   # Parse actions
   244    252                   for iAction, (iActionLine, sAction) in enumerate(lActions, 1):
   245    253                       sAction = sAction.strip()
   246    254                       if sAction:
   247    255                           sActionId = f"{self.sGraphCode}__{sRuleName}__b{iActionBlock}_a{iAction}"
   248         -                        aAction = self.createAction(sActionId, sAction, nPriority, len(lToken), dPos, iActionLine)
          256  +                        aAction = self.createAction(sActionId, sAction, nPriority, len(lToken), dPos, iActionLine, nFirstNullable, nLastNullable)
   249    257                           if aAction:
   250    258                               sActionName = self.storeAction(sActionId, aAction)
   251    259                               lResult = list(lToken)
   252    260                               lResult.extend(["##"+str(iLine), sActionName])
   253    261                               #if iLine == 13341:
   254    262                               #    print("  ".join(lToken))
   255    263                               #    print(sActionId, aAction)
................................................................................
   258    266                               print("# Error on action at line:", iLine)
   259    267                               print(sTokenLine, "\n", lActions)
   260    268                               exit()
   261    269                       else:
   262    270                           print("No action found for ", iActionLine)
   263    271                           exit()
   264    272   
   265         -    def createAction (self, sActionId, sAction, nPriority, nToken, dPos, iActionLine):
          273  +    def createAction (self, sActionId, sAction, nPriority, nToken, dPos, iActionLine, nFirstNullable, nLastNullable):
   266    274           "create action rule as a list"
   267    275           sLineId = "#" + str(iActionLine)
   268    276   
   269    277           # Option
   270    278           sOption = False
   271    279           m = re.match("/(\\w+)/", sAction)
   272    280           if m:
................................................................................
   292    300           # Case sensitivity
   293    301           bCaseSensitivity = not bool(m.group("casing"))
   294    302   
   295    303           # Action
   296    304           cAction = m.group("action")
   297    305           sAction = sAction[m.end():].strip()
   298    306           sAction = changeReferenceToken(sAction, dPos)
          307  +
   299    308           # target
   300    309           cStartLimit = "<"
   301    310           cEndLimit = ">"
   302    311           if not m.group("start"):
   303    312               iStartAction = 1
   304    313               iEndAction = 0
   305    314           else:
................................................................................
   320    329               if iEndAction:
   321    330                   iEndAction = dPos.get(iEndAction, iEndAction)
   322    331           if iStartAction < 0:
   323    332               iStartAction += 1
   324    333           if iEndAction < 0:
   325    334               iEndAction += 1
   326    335   
          336  +        # check target
          337  +        if nFirstNullable > -1:
          338  +            if nFirstNullable > 0 and iStartAction > 0 and iEndAction != 0 and iStartAction > nFirstNullable:
          339  +                print(f"# Error. At {sLineId}, {sActionId}, target start is bigger than first nullable token.")
          340  +            if nFirstNullable > 0 and iEndAction > 0 and iStartAction != 1 and iEndAction > nFirstNullable:
          341  +                print(f"# Error. At {sLineId}, {sActionId}, target end is bigger than first nullable token.")
          342  +            if nLastNullable < 0 and iStartAction < 0 and iEndAction != 0 and (iStartAction-1) < nLastNullable:
          343  +                print(f"# Error. At {sLineId}, {sActionId}, target start is lower than last nullable token.")
          344  +            if nLastNullable < 0 and iEndAction < 0 and iStartAction != 1 and (iEndAction-1) < nLastNullable:
          345  +                print(f"# Error. At {sLineId}, {sActionId}, target end is lower than last nullable token.")
          346  +
   327    347           if cAction == "-":
   328    348               ## error
   329    349               iMsg = sAction.find(" && ")
   330    350               if iMsg == -1:
   331         -                sMsg = "# Error. Error message not found."
   332         -                sURL = ""
   333    351                   print("\n# Error. No message at: ", sLineId, sActionId)
   334    352                   exit()
   335    353               else:
   336    354                   sMsg = sAction[iMsg+4:].strip()
   337    355                   sAction = sAction[:iMsg].strip()
   338    356                   sURL = ""
   339    357                   mURL = re.search("[|] *(https?://.*)", sMsg)
   340    358                   if mURL:
   341    359                       sURL = mURL.group(1).strip()
   342    360                       sMsg = sMsg[:mURL.start(0)].strip()
   343         -                checkTokenNumbers(sMsg, sActionId, nToken)
          361  +                checkTokenNumbers(sMsg, sActionId, nToken)      # check tokens in message
   344    362                   if sMsg[0:1] == "=":
   345    363                       sMsg = self.createFunction("msg", sMsg, True)
   346    364                   else:
   347    365                       checkIfThereIsCode(sMsg, sActionId)
   348    366   
   349         -        # checking consistancy
   350         -        checkTokenNumbers(sAction, sActionId, nToken)
          367  +        # checking token consistancy
          368  +        checkTokenNumbers(sCondition, sActionId, nToken)    # check tokens in condition
          369  +        checkTokenNumbers(sAction, sActionId, nToken)       # check tokens in action
   351    370   
   352    371           if cAction == ">":
   353    372               ## no action, break loop if condition is False
   354    373               return [sLineId, sOption, sCondition, cAction, ""]
   355    374   
   356    375           if not sAction and cAction != "!":
   357    376               print(f"\n# Error in action at line <{sLineId}/{sActionId}>:  This action is empty.")