Grammalecte  Check-in [e31ec0bfe5]

Overview
Comment:[build] save hash of rules file in data cache, automatically use cache if rules file hasn’t been modified
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | build
Files: files | file ages | folders
SHA3-256: e31ec0bfe5d1a5ef8e5f6a21ab51fa83e2bf74c157bf7bc3593638f2a7359e61
User & Date: olr on 2020-04-18 13:30:52
Other Links: manifest | tags
Context
2020-04-18
16:14
[fr] ajustements check-in: 5ae7d109af user: olr tags: fr, trunk
13:30
[build] save hash of rules file in data cache, automatically use cache if rules file hasn’t been modified check-in: e31ec0bfe5 user: olr tags: build, trunk
12:32
[fx] gc panel: tooltip > rework appearance and access to the db check-in: b8188ff0e5 user: olr tags: fx, trunk
Changes

Modified compile_rules.py from [ed8f69534c] to [cec64f3971].

     4      4   
     5      5   import re
     6      6   import os
     7      7   import traceback
     8      8   import json
     9      9   import colorsys
    10     10   import time
           11  +import hashlib
    11     12   
    12     13   import compile_rules_js_convert as jsconv
    13     14   import compile_rules_graph as crg
    14     15   
    15     16   
    16     17   dDEFINITIONS = {}
    17     18   dDECLENSIONS = {}
................................................................................
   467    468       print("  {:>6}:  {}".format(nLine, "  " * nLevel + sComment))
   468    469   
   469    470   
   470    471   def make (spLang, sLang, bUseCache=False):
   471    472       "compile rules, returns a dictionary of values"
   472    473       # for clarity purpose, don’t create any file here
   473    474   
   474         -    if bUseCache and os.path.isfile("_build/data_cache.json"):
   475         -        print("> don’t rebuild rules, use cache...")
          475  +    dCacheVars = None
          476  +
          477  +    if os.path.isfile("_build/data_cache.json"):
          478  +        print("> data cache found")
   476    479           sJSON = open("_build/data_cache.json", "r", encoding="utf-8").read()
   477    480           dCacheVars = json.loads(sJSON)
   478         -        print("  build made at: " + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dCacheVars.get("fBuildTime", 0))))
   479         -        return dCacheVars
   480         -
   481         -    fBuildTime = time.time()
          481  +        sBuildDate = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dCacheVars.get("fBuildTime", 0)))
          482  +        if bUseCache:
          483  +            print("> use cache (no rebuild asked)")
          484  +            print("  build made at: " + sBuildDate)
          485  +            return dCacheVars
   482    486   
   483    487       print("> read rules file...")
   484    488       try:
   485         -        lRules = open(spLang + "/rules.grx", 'r', encoding="utf-8").readlines()
          489  +        sFileContent = open(spLang + "/rules.grx", 'r', encoding="utf-8").read()
   486    490       except OSError:
   487    491           print("Error. Rules file in project [" + sLang + "] not found.")
   488    492           exit()
          493  +
          494  +    xHasher = hashlib.new("sha3_512")
          495  +    xHasher.update(sFileContent.encode("utf-8"))
          496  +    sFileHash = xHasher.hexdigest()
          497  +
          498  +    if dCacheVars and sFileHash == dCacheVars.get("sFileHash", ""):
          499  +        print("> cache hash identical to file hash, use cache")
          500  +        print("  build made at: " + sBuildDate)
          501  +        return dCacheVars
   489    502   
   490    503       # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
   491    504       print("  parsing rules...")
          505  +    fBuildTime = time.time()
   492    506       lRuleLine = []
   493    507       lTest = []
   494    508       lOpt = []
   495    509       bGraph = False
   496    510       lGraphRule = []
   497    511   
   498         -    for i, sLine in enumerate(lRules, 1):
          512  +    for i, sLine in enumerate(sFileContent.split("\n"), 1):
   499    513           if sLine.startswith('#END'):
   500    514               # arbitrary end
   501    515               printBookmark(0, "BREAK BY #END", i)
   502    516               break
   503    517           elif sLine.startswith("#"):
   504    518               # comment
   505    519               pass
................................................................................
   624    638           sJSCallables += "        return " + jsconv.py2js(sReturn) + ";\n"
   625    639           sJSCallables += "    },\n"
   626    640   
   627    641       displayStats(lParagraphRules, lSentenceRules)
   628    642   
   629    643       dVars = {
   630    644           "fBuildTime": fBuildTime,
          645  +        "sFileHash": sFileHash,
   631    646           "callables": sPyCallables,
   632    647           "callablesJS": sJSCallables,
   633    648           "gctests": sGCTests,
   634    649           "gctestsJS": sGCTestsJS,
   635    650           "paragraph_rules": mergeRulesByOption(lParagraphRules),
   636    651           "sentence_rules": mergeRulesByOption(lSentenceRules),
   637    652           "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)),

Modified compile_rules_graph.py from [b7d9668574] to [07d8ba5d53].

   453    453   
   454    454   
   455    455   def make (lRule, sLang, dDef, dDecl, dOptPriority):
   456    456       "compile rules, returns a dictionary of values"
   457    457       # for clarity purpose, don’t create any file here
   458    458   
   459    459       # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
   460         -    print("  parsing rules...")
          460  +    print("  parsing graph rules...")
   461    461       lTokenLine = []
   462    462       lActions = []
   463    463       bActionBlock = False
   464    464       nPriority = -1
   465    465       dAllGraph = {}
   466    466       dGraphCode = {}
   467    467       sGraphName = ""
................................................................................
   541    541               lActions.clear()
   542    542               iActionBlock += 1
   543    543           else:
   544    544               print("Unknown line at:", iLine)
   545    545               print(sLine)
   546    546   
   547    547       # processing rules
   548         -    print("  processing rules...")
          548  +    print("  processing graph rules...")
   549    549       initProcessPoolExecutor()
   550    550       fStartTimer = time.time()
   551    551       # build graph
   552    552       lResult = []
   553    553       nRule = 0
   554    554       for sGraphName, lRuleLine in dAllGraph.items():
   555    555           nRule += len(lRuleLine)