Grammalecte  Check-in [d0bd3382eb]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:[core] update: paragraph splitting
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | core
Files: files | file ages | folders
SHA3-256:d0bd3382ebaefaa81cbeb41789eacce763b4bb638f12820d10e56db40801d00b
User & Date: olr 2019-04-16 16:05:28
Context
2019-04-16
16:29
[fr] faux positif (ocr) check-in: 660df51ea1 user: olr tags: fr, trunk
16:05
[core] update: paragraph splitting check-in: d0bd3382eb user: olr tags: core, trunk
15:58
[fr] faux positifs (ocr) check-in: a052c4803b user: olr tags: fr, trunk
Changes

Changes to gc_core/js/lang_core/gc_engine.js.

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    //// Parsing

    parse: function (sText, sCountry="${country_default}", bDebug=false, dOptions=null, bContext=false) {
        let oText = new TextParser(sText);
        return oText.parse(sCountry, bDebug, dOptions, bContext);
    },

    _zEndOfSentence: new RegExp ('([.?!:;…][   .?!…»”")]+(?=[A-ZÉÈÎÔ])|.$)', "g"),
    _zBeginOfParagraph: new RegExp ("^[-  –—.,;?!…]*", "ig"),
    _zEndOfParagraph: new RegExp ("[-  .,;?!…–—]*$", "ig"),

    getSentenceBoundaries: function* (sText) {
        let mBeginOfSentence = this._zBeginOfParagraph.exec(sText);
        let iStart = this._zBeginOfParagraph.lastIndex;
        let m;







|







161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    //// Parsing

    parse: function (sText, sCountry="${country_default}", bDebug=false, dOptions=null, bContext=false) {
        let oText = new TextParser(sText);
        return oText.parse(sCountry, bDebug, dOptions, bContext);
    },

    _zEndOfSentence: new RegExp ('([.?!:;…][   .?!…»«“”"‘’)]+(?=[A-ZÉÈÎÔ])|.$)', "g"),
    _zBeginOfParagraph: new RegExp ("^[-  –—.,;?!…]*", "ig"),
    _zEndOfParagraph: new RegExp ("[-  .,;?!…–—]*$", "ig"),

    getSentenceBoundaries: function* (sText) {
        let mBeginOfSentence = this._zBeginOfParagraph.exec(sText);
        let iStart = this._zBeginOfParagraph.lastIndex;
        let m;

Changes to gc_core/py/lang_core/gc_engine.py.

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
    "set options to default values"
    global _dOptions
    _dOptions = dict(gc_options.getOptions(_sAppContext))


#### Parsing

_zEndOfSentence = re.compile(r'([.?!:;…][   .?!…»”"’)]+(?=[A-ZÉÈÎÔ])|.$)')
_zBeginOfParagraph = re.compile(r"^\W*")
_zEndOfParagraph = re.compile(r"\W*$")

def _getSentenceBoundaries (sText):
    iStart = _zBeginOfParagraph.match(sText).end()
    for m in _zEndOfSentence.finditer(sText):
        yield (iStart, m.end())







|







187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
    "set options to default values"
    global _dOptions
    _dOptions = dict(gc_options.getOptions(_sAppContext))


#### Parsing

_zEndOfSentence = re.compile(r'([.?!:;…][   .?!…»«“”"‘’)]+(?=[A-ZÉÈÎÔ])|.$)')
_zBeginOfParagraph = re.compile(r"^\W*")
_zEndOfParagraph = re.compile(r"\W*$")

def _getSentenceBoundaries (sText):
    iStart = _zBeginOfParagraph.match(sText).end()
    for m in _zEndOfSentence.finditer(sText):
        yield (iStart, m.end())