Grammalecte  Check-in [8012c45f37]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:[py][core] new parser, [cli] update command list and rename some commands
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | cli | core
Files: files | file ages | folders
SHA3-256:8012c45f374f30c91c46f916b5471726d501e5b0f62f11342e53fb00b5308595
User & Date: olr 2019-05-25 06:53:17
Original Comment: [cli] update command list and rename some commands
Context
2019-05-25
09:51
[fx] update lexicographer CSS check-in: af891c805d user: olr tags: fx, trunk
06:53
[py][core] new parser, [cli] update command list and rename some commands check-in: 8012c45f37 user: olr tags: cli, core, trunk
2019-05-24
14:43
[build][core] update line identifier check-in: b4abcccefb user: olr tags: build, core, trunk
Changes

Changes to gc_core/py/lang_core/gc_engine.py.

229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260

261
262
263
264
265
266
267
268
269
270
271
272















































273
274
275
276
277
278
279
                s += "\t" + str(dToken["aTags"])
            s += "\n"
        #for nPos, dToken in self.dTokenPos.items():
        #    s += "{}\t{}\n".format(nPos, dToken)
        return s

    def parse (self, sCountry="${country_default}", bDebug=False, dOptions=None, bContext=False):
        "analyses the paragraph sText and returns list of errors"
        #sText = unicodedata.normalize("NFC", sText)
        dOpt = dOptions or _dOptions
        bShowRuleId = option('idrule')

        # parse paragraph
        try:
            self.parseText(self.sText, self.sText0, True, 0, sCountry, dOpt, bShowRuleId, bDebug, bContext)
        except:
            raise

        # cleanup
        sText = self.sText
        if " " in sText:
            sText = sText.replace(" ", ' ') # nbsp
        if " " in sText:
            sText = sText.replace(" ", ' ') # nnbsp
        if "'" in sText:
            sText = sText.replace("'", "’")
        if "‑" in sText:
            sText = sText.replace("‑", "-") # nobreakdash
        if "@@" in sText:
            sText = re.sub("@@+", "", sText)

        # parse sentences

        for iStart, iEnd in text.getSentenceBoundaries(sText):
            if 4 < (iEnd - iStart) < 2000:
                try:
                    self.sSentence = sText[iStart:iEnd]
                    self.sSentence0 = self.sText0[iStart:iEnd]
                    self.nOffsetWithinParagraph = iStart
                    self.lToken = list(_oTokenizer.genTokens(self.sSentence, True))
                    self.dTokenPos = { dToken["nStart"]: dToken  for dToken in self.lToken  if dToken["sType"] != "INFO" }
                    self.parseText(self.sSentence, self.sSentence0, False, iStart, sCountry, dOpt, bShowRuleId, bDebug, bContext)
                except:
                    raise
        return self.dError.values() # this is a view (iterable)
















































    def parseText (self, sText, sText0, bParagraph, nOffset, sCountry, dOptions, bShowRuleId, bDebug, bContext):
        "parse the text with rules"
        bChange = False
        for sOption, lRuleGroup in _getRules(bParagraph):
            if sOption == "@@@@":
                # graph rules







|



<





<
<
<
<
<
<
<
<
<
<
<
<
<
<

>












>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







229
230
231
232
233
234
235
236
237
238
239

240
241
242
243
244














245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
                s += "\t" + str(dToken["aTags"])
            s += "\n"
        #for nPos, dToken in self.dTokenPos.items():
        #    s += "{}\t{}\n".format(nPos, dToken)
        return s

    def parse (self, sCountry="${country_default}", bDebug=False, dOptions=None, bContext=False):
        "analyses sText and returns an iterable of errors"
        #sText = unicodedata.normalize("NFC", sText)
        dOpt = dOptions or _dOptions
        bShowRuleId = option('idrule')

        # parse paragraph
        try:
            self.parseText(self.sText, self.sText0, True, 0, sCountry, dOpt, bShowRuleId, bDebug, bContext)
        except:
            raise














        # parse sentences
        sText = self._getCleanText()
        for iStart, iEnd in text.getSentenceBoundaries(sText):
            if 4 < (iEnd - iStart) < 2000:
                try:
                    self.sSentence = sText[iStart:iEnd]
                    self.sSentence0 = self.sText0[iStart:iEnd]
                    self.nOffsetWithinParagraph = iStart
                    self.lToken = list(_oTokenizer.genTokens(self.sSentence, True))
                    self.dTokenPos = { dToken["nStart"]: dToken  for dToken in self.lToken  if dToken["sType"] != "INFO" }
                    self.parseText(self.sSentence, self.sSentence0, False, iStart, sCountry, dOpt, bShowRuleId, bDebug, bContext)
                except:
                    raise
        return self.dError.values() # this is a view (iterable)

    def parseAndGetSentences (self, sCountry="${country_default}", bDebug=False):
        "analyses sText and returns a list of sentences with their tokens"
        #sText = unicodedata.normalize("NFC", sText)
        # parse paragraph
        try:
            self.parseText(self.sText, self.sText0, True, 0, sCountry, dOptions, bShowRuleId, bDebug, bContext)
        except:
            raise
        # parse sentences
        sText = self._getCleanText()
        lSentence = []
        i = 0
        for iStart, iEnd in text.getSentenceBoundaries(sText):
            try:
                self.sSentence = sText[iStart:iEnd]
                self.sSentence0 = self.sText0[iStart:iEnd]
                self.nOffsetWithinParagraph = iStart
                self.lToken = list(_oTokenizer.genTokens(self.sSentence, True))
                self.dTokenPos = { dToken["nStart"]: dToken  for dToken in self.lToken  if dToken["sType"] != "INFO" }
                i += 1
                lSentence.append({
                    "i": i,
                    "iStart": iStart,
                    "iEnd": iEnd,
                    "sSentence": self.sSentence,
                    "sSentence0": self.sSentence0,
                    "lToken": list(lToken) # this is a copy
                })
                self.parseText(self.sSentence, self.sSentence0, False, iStart, sCountry, dOptions, False, False, False)
            except:
                raise
        return lSentence

    def _getCleanText (self):
        sText = self.sText
        if " " in sText:
            sText = sText.replace(" ", ' ') # nbsp
        if " " in sText:
            sText = sText.replace(" ", ' ') # nnbsp
        if "'" in sText:
            sText = sText.replace("'", "’")
        if "‑" in sText:
            sText = sText.replace("‑", "-") # nobreakdash
        if "@@" in sText:
            sText = re.sub("@@+", "", sText)
        return sText

    def parseText (self, sText, sText0, bParagraph, nOffset, sCountry, dOptions, bShowRuleId, bDebug, bContext):
        "parse the text with rules"
        bChange = False
        for sOption, lRuleGroup in _getRules(bParagraph):
            if sOption == "@@@@":
                # graph rules

Changes to grammalecte-cli.py.

21
22
23
24
25
26
27
28
29
30
31
32


33
34
35
36
37
38
39
...
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
_HELP = """
    /help                       /h      show this text
    ?word1 [word2] ...                  words analysis
    !word                               suggestion
    >word                               draw path of word in the word graph
    =filter                             show all entries whose morphology fits to filter
    /lopt                       /lo     list options
    /+ option1 [option2] ...            activate grammar checking options
    /- option1 [option2] ...            deactivate grammar checking options
    /lrules [pattern]           /lr     list rules
    /--rule1 [rule2] ...                deactivate grammar checking rule
    /++rule1 [rule2] ...                reactivate grammar checking rule


    /quit                       /q      exit
"""


def _getText (sInputText):
    sText = input(sInputText)
    if sText == "*":
................................................................................
                    sFlexPattern = sSearch[0:nCut]
                    sTagsPattern = sSearch[nCut+1:]
                else:
                    sFlexPattern = sSearch
                    sTagsPattern = ""
                for aRes in oSpellChecker.select(sFlexPattern, sTagsPattern):
                    echo("\t".join(aRes))
            elif sText.startswith("/+ "):
                oGrammarChecker.gce.setOptions({ opt:True  for opt in sText[3:].strip().split()  if opt in oGrammarChecker.gce.getOptions() })
                echo("done")
            elif sText.startswith("/- "):
                oGrammarChecker.gce.setOptions({ opt:False  for opt in sText[3:].strip().split()  if opt in oGrammarChecker.gce.getOptions() })
                echo("done")
            elif sText.startswith("/-- "):
                for sRule in sText[3:].strip().split():
                    oGrammarChecker.gce.ignoreRule(sRule)
                echo("done")
            elif sText.startswith("/++ "):
                for sRule in sText[3:].strip().split():
                    oGrammarChecker.gce.reactivateRule(sRule)
                echo("done")
            elif sText in ("/debug", "/d"):
                xArgs.debug = not xArgs.debug
                echo("debug mode on"  if xArgs.debug  else "debug mode off")
            elif sText in ("/textformatter", "/tf"):







|
|
|
|
|
>
>







 







|


|


|



|







21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
...
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
_HELP = """
    /help                       /h      show this text
    ?word1 [word2] ...                  words analysis
    !word                               suggestion
    >word                               draw path of word in the word graph
    =filter                             show all entries whose morphology fits to filter
    /lopt                       /lo     list options
    /lrules [pattern]           /lr     list rules
    /o+ option1 [option2] ...           activate grammar checking options
    /o- option1 [option2] ...           deactivate grammar checking options
    /r+ rule1 [rule2] ...               reactivate grammar checking rule
    /r- rule1 [rule2] ...               deactivate grammar checking rule
    /textformatter              /tf     switch on/off the text formatter
    /debug                      /d      switch on/off the debug mode
    /quit                       /q      exit
"""


def _getText (sInputText):
    sText = input(sInputText)
    if sText == "*":
................................................................................
                    sFlexPattern = sSearch[0:nCut]
                    sTagsPattern = sSearch[nCut+1:]
                else:
                    sFlexPattern = sSearch
                    sTagsPattern = ""
                for aRes in oSpellChecker.select(sFlexPattern, sTagsPattern):
                    echo("\t".join(aRes))
            elif sText.startswith("/o+ "):
                oGrammarChecker.gce.setOptions({ opt:True  for opt in sText[3:].strip().split()  if opt in oGrammarChecker.gce.getOptions() })
                echo("done")
            elif sText.startswith("/o- "):
                oGrammarChecker.gce.setOptions({ opt:False  for opt in sText[3:].strip().split()  if opt in oGrammarChecker.gce.getOptions() })
                echo("done")
            elif sText.startswith("/r- "):
                for sRule in sText[3:].strip().split():
                    oGrammarChecker.gce.ignoreRule(sRule)
                echo("done")
            elif sText.startswith("/r+ "):
                for sRule in sText[3:].strip().split():
                    oGrammarChecker.gce.reactivateRule(sRule)
                echo("done")
            elif sText in ("/debug", "/d"):
                xArgs.debug = not xArgs.debug
                echo("debug mode on"  if xArgs.debug  else "debug mode off")
            elif sText in ("/textformatter", "/tf"):