Grammalecte  Check-in [b52cb827b1]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:[core] sentence splitting: code clarification
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | core
Files: files | file ages | folders
SHA3-256:b52cb827b1c8ff0be6d7b814a3c4761e036ce0de04edebf332efe4999caa04ba
User & Date: olr 2019-05-24 14:12:33
Context
2019-05-24
14:43
[build][core] update line identifier check-in: b4abcccefb user: olr tags: build, core, trunk
14:12
[core] sentence splitting: code clarification check-in: b52cb827b1 user: olr tags: core, trunk
12:21
[core] new regex for sentence splitting, generator of sentences check-in: 2777d8cef6 user: olr tags: core, trunk
Changes

Changes to gc_core/js/text.js.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/* global require, exports, console */

"use strict";


var text = {

    _zEndOfSentence: new RegExp ('[.?!:;…]+[   ]+[»”’]?(?=[«"“‘]?[A-ZÉÈÎÔ–—])', "g"),

    getSentenceBoundaries: function* (sText) {
        // generator: returns start and end of sentences found in <sText>
        let iStart = 0;
        let m;
        while ((m = this._zEndOfSentence.exec(sText)) !== null) {
            yield [iStart, this._zEndOfSentence.lastIndex];







|







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/* global require, exports, console */

"use strict";


var text = {

    _zEndOfSentence: new RegExp ('[.?!:;…]+[   ]+[»”’]?(?=[«"“‘–—   ]?[A-ZÉÈÎÔ])', "g"),

    getSentenceBoundaries: function* (sText) {
        // generator: returns start and end of sentences found in <sText>
        let iStart = 0;
        let m;
        while ((m = this._zEndOfSentence.exec(sText)) !== null) {
            yield [iStart, this._zEndOfSentence.lastIndex];

Changes to gc_core/py/text.py.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
"""

import re
import textwrap
from itertools import chain


_zEndOfSentence = re.compile(r'[.?!:;…]+[   ]+[»”’]?(?=[«"“‘]?[A-ZÉÈÎÔ–—])')

def getSentenceBoundaries (sText):
    "generator: returns start and end of sentences found in <sText>"
    iStart = 0
    for m in _zEndOfSentence.finditer(sText):
        yield (iStart, m.end())
        iStart = m.end()







|







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
"""

import re
import textwrap
from itertools import chain


_zEndOfSentence = re.compile(r'[.?!:;…]+[   ]+[»”’]?(?=[«"“‘–—   ]?[A-ZÉÈÎÔ])')

def getSentenceBoundaries (sText):
    "generator: returns start and end of sentences found in <sText>"
    iStart = 0
    for m in _zEndOfSentence.finditer(sText):
        yield (iStart, m.end())
        iStart = m.end()