feat: first release !
This commit is contained in:
Kharec committed
1 parent
77a333e8d9
commit
664527182d
66 files changed
+13812
-16
No files matched your search
@@ -0,0 +1,420 @@
|
||||
package spell
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type letterCase int
|
||||
|
||||
const (
|
||||
lowerCase letterCase = iota
|
||||
titleCase // "Paris"
|
||||
upperCase // "PARIS"
|
||||
mixedCase // "iPhone", "McDonald"
|
||||
)
|
||||
|
||||
func caseOf(w string) letterCase {
|
||||
upper, lower, firstUpper := 0, 0, false
|
||||
for i, r := range w {
|
||||
switch {
|
||||
case unicode.IsUpper(r):
|
||||
upper++
|
||||
if i == 0 {
|
||||
firstUpper = true
|
||||
}
|
||||
case unicode.IsLower(r):
|
||||
lower++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case upper == 0:
|
||||
return lowerCase
|
||||
case lower == 0:
|
||||
return upperCase
|
||||
case upper == 1 && firstUpper:
|
||||
return titleCase
|
||||
}
|
||||
return mixedCase
|
||||
}
|
||||
|
||||
func capitalize(w string) string {
|
||||
r, size := utf8.DecodeRuneInString(w)
|
||||
return string(unicode.ToTitle(r)) + w[size:]
|
||||
}
|
||||
|
||||
// Check reports whether word is spelled correctly. As in Hunspell, a
|
||||
// capitalised or all-caps word is also accepted in its lower-case form
|
||||
// ("Door", "DOOR"), but a lower-case proper noun is not ("paris").
|
||||
func (d *Dictionary) Check(word string) bool {
|
||||
w := d.iconv.Replace(word)
|
||||
if d.accepts(w, false) {
|
||||
return true
|
||||
}
|
||||
switch caseOf(w) {
|
||||
case titleCase:
|
||||
return d.accepts(strings.ToLower(w), true)
|
||||
case upperCase:
|
||||
lower := strings.ToLower(w)
|
||||
return d.accepts(capitalize(lower), true) || d.accepts(lower, true)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// accepts checks one exact spelling, as a word or as a compound word.
|
||||
func (d *Dictionary) accepts(w string, caseChanged bool) bool {
|
||||
if d.valid(w, caseChanged) || (d.compound.enabled() && d.compoundOf(w, 0, "", caseChanged)) {
|
||||
return true
|
||||
}
|
||||
for _, r := range d.compound.rules {
|
||||
if d.matchesRule(w, r, 0, 0, caseChanged) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesRule reports whether w splits into dictionary words (without
|
||||
// affixes, as in Hunspell) whose flags follow rule from step i; parts
|
||||
// counts the words already matched.
|
||||
func (d *Dictionary) matchesRule(w string, rule compoundRule, i, parts int, cc bool) bool {
|
||||
if i < len(rule) && rule[i].rep != 0 && d.matchesRule(w, rule, i+1, parts, cc) {
|
||||
return true // an optional step left out
|
||||
}
|
||||
if w == "" {
|
||||
return i == len(rule) && parts >= 2
|
||||
}
|
||||
if i == len(rule) {
|
||||
return false
|
||||
}
|
||||
step := rule[i]
|
||||
next := i + 1
|
||||
if step.rep == '*' {
|
||||
next = i
|
||||
}
|
||||
runes := []rune(w)
|
||||
for n := d.compound.min; n <= len(runes); n++ {
|
||||
part := string(runes[:n])
|
||||
if d.isForbidden(part) || !d.stemHas(part, cc, func(f flagSet) bool { return f.has(step.flag) }) {
|
||||
continue
|
||||
}
|
||||
if d.matchesRule(string(runes[n:]), rule, next, parts+1, cc) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Where a part sits in a compound word.
|
||||
const (
|
||||
partBegin = iota
|
||||
partMiddle
|
||||
partLast
|
||||
)
|
||||
|
||||
// compoundOf reports whether w splits into compound parts, the first of
|
||||
// which follows parts earlier ones (prev is the last of them).
|
||||
func (d *Dictionary) compoundOf(w string, parts int, prev string, cc bool) bool {
|
||||
c := d.compound
|
||||
runes := []rune(w)
|
||||
fits := func(n int) bool { return c.maxWords == 0 || n <= c.maxWords }
|
||||
for i := c.min; i <= len(runes)-c.min; i++ {
|
||||
head, rest := string(runes[:i]), string(runes[i:])
|
||||
if c.checkDup && head == prev {
|
||||
continue
|
||||
}
|
||||
if c.checkCase && (unicode.IsUpper(runes[i]) || unicode.IsUpper(runes[i-1])) {
|
||||
continue
|
||||
}
|
||||
pos := partBegin
|
||||
if parts > 0 {
|
||||
pos = partMiddle
|
||||
}
|
||||
if !d.compoundPart(head, pos, cc) {
|
||||
continue
|
||||
}
|
||||
if fits(parts+2) && !(c.checkDup && rest == head) && d.compoundPart(rest, partLast, cc) {
|
||||
return true
|
||||
}
|
||||
if fits(parts+3) && d.compoundOf(rest, parts+1, head, cc) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// compoundPart checks one part of a compound word. The first part may take
|
||||
// a prefix and the last one a suffix; other affixes need COMPOUNDPERMITFLAG.
|
||||
func (d *Dictionary) compoundPart(w string, pos int, cc bool) bool {
|
||||
c := d.compound
|
||||
allowed := func(f flagSet) bool {
|
||||
return f.has(c.any) || (pos == partBegin && f.has(c.begin)) ||
|
||||
(pos == partMiddle && f.has(c.middle)) || (pos == partLast && f.has(c.last))
|
||||
}
|
||||
if d.isForbidden(w) {
|
||||
return false
|
||||
}
|
||||
if d.stemHas(w, cc, func(f flagSet) bool { return !f.has(d.needAffix) && allowed(f) }) {
|
||||
return true
|
||||
}
|
||||
affixOK := func(a *affix, at int) bool {
|
||||
return !a.cont.has(c.forbid) && (pos == at || a.cont.has(c.permit))
|
||||
}
|
||||
if d.matchingSuffixes(w, func(s *affix, stem string) bool {
|
||||
return affixOK(s, partLast) && d.stemHas(stem, cc, func(f flagSet) bool { return f.has(s.flag) && allowed(f) })
|
||||
}) {
|
||||
return true
|
||||
}
|
||||
return d.matchingPrefixes(w, func(p *affix, rest string) bool {
|
||||
if !affixOK(p, partBegin) {
|
||||
return false
|
||||
}
|
||||
if d.stemHas(rest, cc, func(f flagSet) bool { return f.has(p.flag) && allowed(f) }) {
|
||||
return true
|
||||
}
|
||||
// Both: a prefix and a suffix that combine (cross product).
|
||||
return p.cross && d.matchingSuffixes(rest, func(s *affix, stem string) bool {
|
||||
return s.cross && affixOK(s, partLast) &&
|
||||
d.stemHas(stem, cc, func(f flagSet) bool { return f.has(p.flag) && f.has(s.flag) && allowed(f) })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// valid checks one exact spelling. caseChanged is set when the word was
|
||||
// lower-cased to be looked up, which KEEPCASE words forbid.
|
||||
func (d *Dictionary) valid(w string, caseChanged bool) bool {
|
||||
if w == "" || d.isForbidden(w) {
|
||||
return false
|
||||
}
|
||||
if d.stemHas(w, caseChanged, func(f flagSet) bool { return !f.has(d.needAffix) && !f.has(d.onlyInComp) }) {
|
||||
return true
|
||||
}
|
||||
return d.suffixed(w, caseChanged) || d.prefixed(w, caseChanged)
|
||||
}
|
||||
|
||||
func (d *Dictionary) isForbidden(w string) bool {
|
||||
if d.forbidden == 0 {
|
||||
return false
|
||||
}
|
||||
for _, f := range d.words[w] {
|
||||
if f.has(d.forbidden) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// stemHas reports whether one of the homonyms of stem satisfies ok.
|
||||
func (d *Dictionary) stemHas(stem string, caseChanged bool, ok func(flagSet) bool) bool {
|
||||
for _, f := range d.words[stem] {
|
||||
if f.has(d.forbidden) || (caseChanged && f.has(d.keepCase)) {
|
||||
continue
|
||||
}
|
||||
if ok(f) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchingSuffixes calls fn for every suffix that w can end with, with the
|
||||
// stem it leaves once the suffix is removed and the stripped text restored.
|
||||
func (d *Dictionary) matchingSuffixes(w string, fn func(a *affix, stem string) bool) bool {
|
||||
for n := 0; n <= min(d.maxSuffix, len(w)); n++ {
|
||||
for _, a := range d.suffixes[w[len(w)-n:]] {
|
||||
base := w[:len(w)-n]
|
||||
if base == "" && !d.fullStrip {
|
||||
continue
|
||||
}
|
||||
stem := base + a.strip
|
||||
if stem == "" || !a.cond.matchEnd(stem) {
|
||||
continue
|
||||
}
|
||||
if fn(a, stem) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Dictionary) matchingPrefixes(w string, fn func(a *affix, stem string) bool) bool {
|
||||
for n := 0; n <= min(d.maxPrefix, len(w)); n++ {
|
||||
for _, a := range d.prefixes[w[:n]] {
|
||||
rest := w[n:]
|
||||
if rest == "" && !d.fullStrip {
|
||||
continue
|
||||
}
|
||||
stem := a.strip + rest
|
||||
if stem == "" || !a.cond.matchStart(stem) {
|
||||
continue
|
||||
}
|
||||
if fn(a, stem) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// suffixed accepts stem+suffix, and stem+suffix+suffix when the inner
|
||||
// suffix lists the outer one as a continuation class.
|
||||
func (d *Dictionary) suffixed(w string, cc bool) bool {
|
||||
return d.matchingSuffixes(w, func(s *affix, stem string) bool {
|
||||
if !s.cont.has(d.needAffix) && !s.cont.has(d.circumfix) &&
|
||||
d.stemHas(stem, cc, func(f flagSet) bool { return f.has(s.flag) }) {
|
||||
return true
|
||||
}
|
||||
if !d.contFlags[s.flag] {
|
||||
return false
|
||||
}
|
||||
return d.matchingSuffixes(stem, func(inner *affix, root string) bool {
|
||||
return inner.cont.has(s.flag) &&
|
||||
d.stemHas(root, cc, func(f flagSet) bool { return f.has(inner.flag) })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// prefixed accepts prefix+stem and prefix+stem+suffix.
|
||||
func (d *Dictionary) prefixed(w string, cc bool) bool {
|
||||
return d.matchingPrefixes(w, func(p *affix, rest string) bool {
|
||||
if !p.cont.has(d.needAffix) && !p.cont.has(d.circumfix) &&
|
||||
d.stemHas(rest, cc, func(f flagSet) bool { return f.has(p.flag) }) {
|
||||
return true
|
||||
}
|
||||
return d.matchingSuffixes(rest, func(s *affix, stem string) bool {
|
||||
if s.cont.has(d.circumfix) != p.cont.has(d.circumfix) {
|
||||
return false
|
||||
}
|
||||
return d.stemHas(stem, cc, func(f flagSet) bool {
|
||||
switch {
|
||||
case f.has(s.flag) && f.has(p.flag):
|
||||
return p.cross && s.cross
|
||||
case f.has(s.flag):
|
||||
return s.cont.has(p.flag) // the suffix allows the prefix
|
||||
case f.has(p.flag):
|
||||
return p.cont.has(s.flag)
|
||||
}
|
||||
return false
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// maxSuggestions is how many corrections Suggest returns at most.
|
||||
const maxSuggestions = 8
|
||||
|
||||
// Suggest returns likely corrections for a misspelled word, best first,
|
||||
// in the word's case ("TEH" gives "THE"). It tries the word with a capital
|
||||
// (proper nouns), the dictionary's table of common misspellings (REP), then
|
||||
// every spelling one edit away, the most frequent kinds of typo first:
|
||||
// swapped letters, an extra letter, a wrong letter, a missing letter, and
|
||||
// a missing space between two words of three letters or more.
|
||||
func (d *Dictionary) Suggest(word string) []string {
|
||||
w := d.iconv.Replace(word)
|
||||
wc := caseOf(w)
|
||||
base := w
|
||||
if wc == titleCase || wc == upperCase {
|
||||
base = strings.ToLower(w)
|
||||
}
|
||||
|
||||
var out, proper []string
|
||||
seen := map[string]bool{w: true}
|
||||
try := func(cand string) {
|
||||
if len(out) >= maxSuggestions || seen[cand] {
|
||||
return
|
||||
}
|
||||
seen[cand] = true
|
||||
if d.suggestable(cand) {
|
||||
out = append(out, cand)
|
||||
return
|
||||
}
|
||||
// A lower-case candidate may be a proper noun ("pariss" → "Paris"),
|
||||
// offered after the common words.
|
||||
if caseOf(cand) == lowerCase {
|
||||
if cand = capitalize(cand); !seen[cand] && d.suggestable(cand) {
|
||||
seen[cand] = true
|
||||
proper = append(proper, cand)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if wc == lowerCase && d.suggestable(capitalize(base)) {
|
||||
out = append(out, capitalize(base))
|
||||
}
|
||||
for _, r := range d.rep {
|
||||
for i := 0; ; {
|
||||
j := strings.Index(base[i:], r.from)
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
at := i + j
|
||||
if (!r.start || at == 0) && (!r.end || at+len(r.from) == len(base)) {
|
||||
try(base[:at] + r.to + base[at+len(r.from):])
|
||||
}
|
||||
i = at + 1
|
||||
}
|
||||
}
|
||||
|
||||
runes := []rune(base)
|
||||
edit := func(i, del int, ins ...rune) string {
|
||||
return string(runes[:i]) + string(ins) + string(runes[i+del:])
|
||||
}
|
||||
for i := 0; i+1 < len(runes); i++ {
|
||||
if runes[i] != runes[i+1] {
|
||||
try(edit(i, 2, runes[i+1], runes[i]))
|
||||
}
|
||||
}
|
||||
for i := range runes {
|
||||
try(edit(i, 1))
|
||||
}
|
||||
for i := range runes {
|
||||
for _, c := range d.try {
|
||||
if c != runes[i] && !unicode.IsUpper(c) {
|
||||
try(edit(i, 1, c))
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := 0; i <= len(runes); i++ {
|
||||
for _, c := range d.try {
|
||||
if !unicode.IsUpper(c) {
|
||||
try(edit(i, 0, c))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Short words ("a", "re", "er") make most splits look valid but are
|
||||
// rarely what was meant; REP tables cover the common cases ("a lot").
|
||||
for i := 3; i+3 <= len(runes); i++ {
|
||||
try(edit(i, 0, ' '))
|
||||
}
|
||||
|
||||
out = append(out, proper...)
|
||||
out = out[:min(len(out), maxSuggestions)]
|
||||
for i, s := range out {
|
||||
switch wc {
|
||||
case upperCase:
|
||||
s = strings.ToUpper(s)
|
||||
case titleCase:
|
||||
s = capitalize(s)
|
||||
}
|
||||
out[i] = d.oconv.Replace(s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// suggestable checks a candidate, which may be two words separated by a
|
||||
// space, and leaves out words marked NOSUGGEST (offensive words, mostly).
|
||||
func (d *Dictionary) suggestable(cand string) bool {
|
||||
for _, w := range strings.Split(cand, " ") {
|
||||
if w == "" || strings.HasPrefix(w, "-") || strings.HasSuffix(w, "-") || !d.valid(w, false) {
|
||||
return false
|
||||
}
|
||||
for _, f := range d.words[w] {
|
||||
if f.has(d.noSuggest) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package spell
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
|
||||
"plotzilla/internal/screenplay"
|
||||
)
|
||||
|
||||
// Word is a word found in a text. Offset and Length count runes, like
|
||||
// every offset exchanged with the editor.
|
||||
type Word struct {
|
||||
Text string
|
||||
Offset int
|
||||
Length int
|
||||
}
|
||||
|
||||
func isApostrophe(r rune) bool { return r == '\'' || r == '’' }
|
||||
|
||||
func isWordRune(r rune) bool {
|
||||
return unicode.IsLetter(r) || unicode.IsMark(r) || unicode.IsDigit(r)
|
||||
}
|
||||
|
||||
// Words splits text into words: runs of letters and digits, with the
|
||||
// apostrophes inside them ("don't", "l'homme"). Hyphens, slashes and
|
||||
// dots separate words, so each half of "well-known" is checked alone.
|
||||
func Words(text string) []Word {
|
||||
var words []Word
|
||||
runes := []rune(text)
|
||||
for i := 0; i < len(runes); {
|
||||
if !isWordRune(runes[i]) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
start := i
|
||||
for i < len(runes) && (isWordRune(runes[i]) ||
|
||||
(isApostrophe(runes[i]) && i+1 < len(runes) && isWordRune(runes[i+1]))) {
|
||||
i++
|
||||
}
|
||||
words = append(words, Word{Text: string(runes[start:i]), Offset: start, Length: i - start})
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
// Misspelling locates a misspelled word: the index of its element in
|
||||
// document order (as layout.Break) and its rune offset and length.
|
||||
type Misspelling struct {
|
||||
Element int `json:"element"`
|
||||
Offset int `json:"offset"`
|
||||
Length int `json:"length"`
|
||||
Word string `json:"word"`
|
||||
}
|
||||
|
||||
// jargon lists screenplay abbreviations that no dictionary knows.
|
||||
var jargon = map[string]bool{
|
||||
"int": true, "ext": true, "est": true, "cont'd": true, "contd": true,
|
||||
"pov": true, "vo": true, "os": true, "oc": true, "ecu": true, "mos": true,
|
||||
}
|
||||
|
||||
var sceneOpener = regexp.MustCompile(`(?i)^\s*(int\.?/ext|ext\.?/int|int|ext|est|i/e)\b`)
|
||||
|
||||
// Checker checks screenplays against one dictionary plus the user's own
|
||||
// words, and remembers the answers: a script repeats the same words.
|
||||
type Checker struct {
|
||||
dict *Dictionary
|
||||
mu sync.Mutex
|
||||
cache map[string]bool
|
||||
personal map[string]bool // lower-case
|
||||
}
|
||||
|
||||
func NewChecker(d *Dictionary, personal []string) *Checker {
|
||||
c := &Checker{dict: d, cache: map[string]bool{}, personal: map[string]bool{}}
|
||||
for _, w := range personal {
|
||||
c.personal[strings.ToLower(w)] = true
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Add accepts word from now on, in any letter case.
|
||||
func (c *Checker) Add(word string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.personal[strings.ToLower(normalize(word))] = true
|
||||
}
|
||||
|
||||
func normalize(w string) string { return strings.ReplaceAll(w, "’", "'") }
|
||||
|
||||
// Check reports whether word is correct. Words with digits ("3rd", "R2D2")
|
||||
// and single letters are always accepted.
|
||||
func (c *Checker) Check(word string) bool {
|
||||
if len([]rune(word)) < 2 || strings.IndexFunc(word, unicode.IsDigit) >= 0 {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(normalize(word))
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if jargon[lower] || c.personal[lower] {
|
||||
return true
|
||||
}
|
||||
ok, found := c.cache[word]
|
||||
if !found {
|
||||
ok = c.dict.Check(word)
|
||||
c.cache[word] = ok
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
func (c *Checker) Suggest(word string) []string { return c.dict.Suggest(word) }
|
||||
|
||||
// Misspellings returns the misspelled words of a script. Character cues
|
||||
// hold names, so they are not checked, and the names they introduce are
|
||||
// accepted everywhere else (an action line often spells them in capitals).
|
||||
func (c *Checker) Misspellings(doc *screenplay.Document) []Misspelling {
|
||||
names := map[string]bool{}
|
||||
for _, el := range doc.Elements {
|
||||
if el.Type == screenplay.Character {
|
||||
for _, w := range Words(screenplay.CharacterName(el.PrintedText())) {
|
||||
names[strings.ToLower(normalize(w.Text))] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := []Misspelling{}
|
||||
for i, el := range doc.Elements {
|
||||
if el.Type == screenplay.Character {
|
||||
continue
|
||||
}
|
||||
text := el.Text()
|
||||
skip := 0 // runes of a scene heading's "INT./EXT."
|
||||
if el.Type == screenplay.Scene {
|
||||
if m := sceneOpener.FindString(text); m != "" {
|
||||
skip = len([]rune(m))
|
||||
}
|
||||
}
|
||||
for _, w := range Words(text) {
|
||||
if w.Offset < skip || isName(names, w.Text) || c.Check(w.Text) {
|
||||
continue
|
||||
}
|
||||
out = append(out, Misspelling{Element: i, Offset: w.Offset, Length: w.Length, Word: w.Text})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isName reports whether w is one of the names, possibly possessive
|
||||
// ("ELODIE's").
|
||||
func isName(names map[string]bool, w string) bool {
|
||||
lower := strings.ToLower(normalize(w))
|
||||
return names[lower] || names[strings.TrimSuffix(lower, "'s")]
|
||||
}
|
||||
|
||||
// Language is an installed dictionary, named after its files ("en_US").
|
||||
type Language struct {
|
||||
Code string
|
||||
Aff string
|
||||
Dic string
|
||||
}
|
||||
|
||||
// Available lists the dictionaries found in dirs; when the same language
|
||||
// is in several directories, the first one wins.
|
||||
func Available(dirs []string) []Language {
|
||||
seen := map[string]bool{}
|
||||
var out []Language
|
||||
for _, dir := range dirs {
|
||||
matches, _ := filepath.Glob(filepath.Join(dir, "*.dic"))
|
||||
sort.Strings(matches)
|
||||
for _, dic := range matches {
|
||||
code := strings.TrimSuffix(filepath.Base(dic), ".dic")
|
||||
aff := strings.TrimSuffix(dic, ".dic") + ".aff"
|
||||
if seen[code] || strings.HasPrefix(code, "hyph_") || strings.HasPrefix(code, "th_") {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(aff); err != nil {
|
||||
continue
|
||||
}
|
||||
seen[code] = true
|
||||
out = append(out, Language{Code: code, Aff: aff, Dic: dic})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Preferred picks the dictionary that suits a locale such as
|
||||
// "fr_FR.UTF-8": the same language and region, then the same language,
|
||||
// then American English, then the first one.
|
||||
func Preferred(codes []string, locale string) string {
|
||||
if len(codes) == 0 {
|
||||
return ""
|
||||
}
|
||||
locale, _, _ = strings.Cut(locale, ".")
|
||||
locale, _, _ = strings.Cut(locale, "@")
|
||||
lang, _, _ := strings.Cut(locale, "_")
|
||||
for _, want := range []func(string) bool{
|
||||
func(c string) bool { return locale != "" && c == locale },
|
||||
func(c string) bool { return lang != "" && (c == lang || strings.HasPrefix(c, lang+"_")) },
|
||||
func(c string) bool { return c == "en_US" },
|
||||
func(c string) bool { return strings.HasPrefix(c, "en_") || c == "en" },
|
||||
} {
|
||||
for _, c := range codes {
|
||||
if want(c) {
|
||||
return c
|
||||
}
|
||||
}
|
||||
}
|
||||
return codes[0]
|
||||
}
|
||||
@@ -0,0 +1,555 @@
|
||||
// Package spell checks spelling with Hunspell dictionaries (the .aff and
|
||||
// .dic files shipped by Linux distributions and LibreOffice). It implements
|
||||
// the part of the Hunspell format that ordinary words need: prefixes and
|
||||
// suffixes with their conditions, cross products, two-level suffixes,
|
||||
// flag aliases and the usual word flags, plus compound words built with
|
||||
// COMPOUNDFLAG or COMPOUNDBEGIN/MIDDLE/LAST (German, Dutch, Scandinavian
|
||||
// dictionaries) or described by COMPOUNDRULE patterns.
|
||||
package spell
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type flag uint16
|
||||
|
||||
// flagSet is a sorted list of flags; words and affixes carry only a few.
|
||||
type flagSet []flag
|
||||
|
||||
func (s flagSet) has(f flag) bool {
|
||||
if f == 0 {
|
||||
return false
|
||||
}
|
||||
i := sort.Search(len(s), func(i int) bool { return s[i] >= f })
|
||||
return i < len(s) && s[i] == f
|
||||
}
|
||||
|
||||
// condition is a Hunspell affix condition: a sequence of characters, "."
|
||||
// or bracket classes ("[^aeiou]y"), matched against one end of the stem.
|
||||
type condition []condUnit
|
||||
|
||||
type condUnit struct {
|
||||
any bool
|
||||
neg bool
|
||||
chars string
|
||||
}
|
||||
|
||||
func (u condUnit) match(r rune) bool {
|
||||
if u.any {
|
||||
return true
|
||||
}
|
||||
return strings.ContainsRune(u.chars, r) != u.neg
|
||||
}
|
||||
|
||||
func parseCondition(s string) condition {
|
||||
if s == "" || s == "." {
|
||||
return nil
|
||||
}
|
||||
var c condition
|
||||
runes := []rune(s)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
switch runes[i] {
|
||||
case '.':
|
||||
c = append(c, condUnit{any: true})
|
||||
case '[':
|
||||
u := condUnit{}
|
||||
i++
|
||||
if i < len(runes) && runes[i] == '^' {
|
||||
u.neg = true
|
||||
i++
|
||||
}
|
||||
start := i
|
||||
for i < len(runes) && runes[i] != ']' {
|
||||
i++
|
||||
}
|
||||
u.chars = string(runes[start:i])
|
||||
c = append(c, u)
|
||||
default:
|
||||
c = append(c, condUnit{chars: string(runes[i])})
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c condition) matchStart(s string) bool {
|
||||
for _, u := range c {
|
||||
r, size := utf8.DecodeRuneInString(s)
|
||||
if size == 0 || !u.match(r) {
|
||||
return false
|
||||
}
|
||||
s = s[size:]
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c condition) matchEnd(s string) bool {
|
||||
for i := len(c) - 1; i >= 0; i-- {
|
||||
r, size := utf8.DecodeLastRuneInString(s)
|
||||
if size == 0 || !c[i].match(r) {
|
||||
return false
|
||||
}
|
||||
s = s[:len(s)-size]
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type affix struct {
|
||||
flag flag
|
||||
cross bool
|
||||
strip string
|
||||
add string
|
||||
cond condition
|
||||
cont flagSet // continuation classes: affixes allowed on top of this one
|
||||
}
|
||||
|
||||
// compounding holds the rules for words made of several dictionary words
|
||||
// ("Haustür" = "Haus" + "Tür").
|
||||
type compounding struct {
|
||||
any, begin, middle, last flag // COMPOUNDFLAG, COMPOUNDBEGIN, COMPOUNDMIDDLE, COMPOUNDLAST
|
||||
permit, forbid flag // affixes allowed or forbidden inside a compound
|
||||
min int // shortest part, in characters (COMPOUNDMIN, 3 by default)
|
||||
maxWords int // most parts (COMPOUNDWORDMAX, 0 = no limit)
|
||||
checkDup bool // CHECKCOMPOUNDDUP: no part repeated twice in a row
|
||||
checkCase bool // CHECKCOMPOUNDCASE: no capital letter at a boundary
|
||||
rules []compoundRule
|
||||
}
|
||||
|
||||
func (c compounding) enabled() bool { return c.any != 0 || c.begin != 0 || c.last != 0 }
|
||||
|
||||
// compoundRule is a COMPOUNDRULE pattern: a sequence of flags, each
|
||||
// optionally followed by "*" (any number of parts) or "?" (at most one).
|
||||
// A compound matches when its parts, in order, carry those flags.
|
||||
type compoundRule []ruleStep
|
||||
|
||||
type ruleStep struct {
|
||||
flag flag
|
||||
rep byte // 0, '*' or '?'
|
||||
}
|
||||
|
||||
// parseRule reads a pattern such as "n*1t" or, with long or numeric flags,
|
||||
// "(aa)(bb)*".
|
||||
func (d *Dictionary) parseRule(s string) compoundRule {
|
||||
var r compoundRule
|
||||
runes := []rune(s)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
name := string(runes[i])
|
||||
if runes[i] == '(' {
|
||||
end := i + 1
|
||||
for end < len(runes) && runes[end] != ')' {
|
||||
end++
|
||||
}
|
||||
name = string(runes[i+1 : min(end, len(runes))])
|
||||
i = end
|
||||
}
|
||||
if name == "*" || name == "?" || name == "" {
|
||||
continue
|
||||
}
|
||||
step := ruleStep{flag: d.flagID(name)}
|
||||
if i+1 < len(runes) && (runes[i+1] == '*' || runes[i+1] == '?') {
|
||||
step.rep = byte(runes[i+1])
|
||||
i++
|
||||
}
|
||||
r = append(r, step)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
type replacement struct {
|
||||
from, to string
|
||||
start bool // "^" anchor
|
||||
end bool // "$" anchor
|
||||
}
|
||||
|
||||
// Dictionary is a loaded Hunspell dictionary. It is safe for concurrent
|
||||
// use once loaded.
|
||||
type Dictionary struct {
|
||||
words map[string][]flagSet // homonyms keep separate flag sets
|
||||
|
||||
// Affixes are indexed by the text they add, so a word is only tested
|
||||
// against the affixes that can have produced it.
|
||||
prefixes map[string][]*affix
|
||||
suffixes map[string][]*affix
|
||||
maxPrefix int // longest added text, in bytes
|
||||
maxSuffix int
|
||||
contFlags map[flag]bool // suffix flags used as a continuation class
|
||||
fullStrip bool
|
||||
needAffix flag
|
||||
forbidden flag
|
||||
keepCase flag
|
||||
noSuggest flag
|
||||
onlyInComp flag
|
||||
circumfix flag
|
||||
compound compounding
|
||||
try []rune
|
||||
rep []replacement
|
||||
iconv *strings.Replacer
|
||||
oconv *strings.Replacer
|
||||
flagMode string
|
||||
flagIDs map[string]flag
|
||||
flagAliases []flagSet
|
||||
}
|
||||
|
||||
// Load reads a dictionary from its .aff and .dic files.
|
||||
func Load(affPath, dicPath string) (*Dictionary, error) {
|
||||
aff, err := os.ReadFile(affPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading affix file: %w", err)
|
||||
}
|
||||
dic, err := os.ReadFile(dicPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading dictionary: %w", err)
|
||||
}
|
||||
return Parse(aff, dic)
|
||||
}
|
||||
|
||||
// Parse builds a dictionary from the contents of .aff and .dic files, in
|
||||
// the encoding named by the affix file's SET directive.
|
||||
func Parse(aff, dic []byte) (*Dictionary, error) {
|
||||
enc := encodingOf(aff)
|
||||
affText, err := decode(aff, enc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dicText, err := decode(dic, enc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &Dictionary{
|
||||
compound: compounding{min: 3},
|
||||
words: map[string][]flagSet{},
|
||||
prefixes: map[string][]*affix{},
|
||||
suffixes: map[string][]*affix{},
|
||||
contFlags: map[flag]bool{},
|
||||
flagIDs: map[string]flag{},
|
||||
}
|
||||
if err := d.parseAff(affText); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.parseDic(dicText)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func encodingOf(aff []byte) string {
|
||||
for _, line := range bytes.Split(aff, []byte("\n")) {
|
||||
if f := bytes.Fields(line); len(f) >= 2 && string(f[0]) == "SET" {
|
||||
return strings.ToUpper(string(f[1]))
|
||||
}
|
||||
}
|
||||
return "ISO8859-1" // Hunspell's default
|
||||
}
|
||||
|
||||
// ISO 8859-15 differs from Latin-1 in eight positions (€, Š, Ž, Œ…).
|
||||
var latin9 = map[byte]rune{
|
||||
0xA4: '€', 0xA6: 'Š', 0xA8: 'š', 0xB4: 'Ž', 0xB8: 'ž', 0xBC: 'Œ', 0xBD: 'œ', 0xBE: 'Ÿ',
|
||||
}
|
||||
|
||||
func decode(b []byte, enc string) (string, error) {
|
||||
switch strings.ReplaceAll(enc, "-", "") {
|
||||
case "UTF8":
|
||||
return strings.TrimPrefix(string(b), "\uFEFF"), nil
|
||||
case "ISO88591", "ISO885915":
|
||||
isLatin9 := strings.HasSuffix(enc, "15")
|
||||
var sb strings.Builder
|
||||
sb.Grow(len(b))
|
||||
for _, c := range b {
|
||||
r, ok := latin9[c]
|
||||
if !ok || !isLatin9 {
|
||||
r = rune(c) // Latin-1 bytes are the first 256 code points
|
||||
}
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
return sb.String(), nil
|
||||
}
|
||||
return "", fmt.Errorf("unsupported dictionary encoding %s", enc)
|
||||
}
|
||||
|
||||
// flagID interns a flag name ("A", "L'", "123") as a small integer.
|
||||
func (d *Dictionary) flagID(name string) flag {
|
||||
if id, ok := d.flagIDs[name]; ok {
|
||||
return id
|
||||
}
|
||||
id := flag(len(d.flagIDs) + 1)
|
||||
d.flagIDs[name] = id
|
||||
return id
|
||||
}
|
||||
|
||||
// parseFlags reads a flag field, which is either an alias number (AF) or
|
||||
// flags written in the dictionary's FLAG format.
|
||||
func (d *Dictionary) parseFlags(s string) flagSet {
|
||||
if len(d.flagAliases) > 0 {
|
||||
if n, err := strconv.Atoi(s); err == nil {
|
||||
if n >= 1 && n <= len(d.flagAliases) {
|
||||
return d.flagAliases[n-1]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return d.plainFlags(s)
|
||||
}
|
||||
|
||||
// plainFlags reads flags written in the dictionary's FLAG format.
|
||||
func (d *Dictionary) plainFlags(s string) flagSet {
|
||||
var names []string
|
||||
switch d.flagMode {
|
||||
case "long":
|
||||
r := []rune(s)
|
||||
for i := 0; i+1 < len(r); i += 2 {
|
||||
names = append(names, string(r[i:i+2]))
|
||||
}
|
||||
case "num":
|
||||
names = strings.Split(s, ",")
|
||||
default: // single characters, "UTF-8" included
|
||||
for _, r := range s {
|
||||
names = append(names, string(r))
|
||||
}
|
||||
}
|
||||
set := make(flagSet, 0, len(names))
|
||||
for _, n := range names {
|
||||
if n = strings.TrimSpace(n); n != "" {
|
||||
set = append(set, d.flagID(n))
|
||||
}
|
||||
}
|
||||
sort.Slice(set, func(i, j int) bool { return set[i] < set[j] })
|
||||
return set
|
||||
}
|
||||
|
||||
func (d *Dictionary) oneFlag(s string) flag {
|
||||
if f := d.parseFlags(s); len(f) > 0 {
|
||||
return f[0]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// pairs collects the pairs of a table directive such as ICONV or REP.
|
||||
func pairs(lines [][]string, key string) [][2]string {
|
||||
var out [][2]string
|
||||
for _, f := range lines {
|
||||
if len(f) >= 3 && f[0] == key {
|
||||
out = append(out, [2]string{f[1], f[2]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (d *Dictionary) parseAff(text string) error {
|
||||
var lines [][]string
|
||||
sc := bufio.NewScanner(strings.NewReader(text))
|
||||
sc.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
f := strings.Fields(sc.Text())
|
||||
if len(f) == 0 || strings.HasPrefix(f[0], "#") {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, f)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return fmt.Errorf("reading affix file: %w", err)
|
||||
}
|
||||
|
||||
// FLAG and AF change how every other flag is read: take them first.
|
||||
for _, f := range lines {
|
||||
if f[0] == "FLAG" && len(f) >= 2 {
|
||||
d.flagMode = f[1]
|
||||
}
|
||||
}
|
||||
var aliases []flagSet
|
||||
counted := false
|
||||
for _, f := range lines {
|
||||
if f[0] == "AF" && len(f) >= 2 {
|
||||
if !counted {
|
||||
counted = true // the first line holds the number of aliases
|
||||
continue
|
||||
}
|
||||
aliases = append(aliases, d.plainFlags(f[1]))
|
||||
}
|
||||
}
|
||||
d.flagAliases = aliases
|
||||
|
||||
// COMPOUNDRULE: the first line holds the number of rules.
|
||||
counted = false
|
||||
for _, f := range lines {
|
||||
if f[0] == "COMPOUNDRULE" && len(f) >= 2 {
|
||||
if !counted {
|
||||
counted = true
|
||||
continue
|
||||
}
|
||||
if r := d.parseRule(f[1]); len(r) > 0 {
|
||||
d.compound.rules = append(d.compound.rules, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pending := map[string]int{} // affix entries still expected per "PFX A"
|
||||
cross := map[string]bool{}
|
||||
for _, f := range lines {
|
||||
switch f[0] {
|
||||
case "TRY":
|
||||
if len(f) >= 2 {
|
||||
d.try = []rune(f[1])
|
||||
}
|
||||
case "FULLSTRIP":
|
||||
d.fullStrip = true
|
||||
case "NEEDAFFIX", "PSEUDOROOT":
|
||||
d.needAffix = d.flagArg(f)
|
||||
case "FORBIDDENWORD":
|
||||
d.forbidden = d.flagArg(f)
|
||||
case "KEEPCASE":
|
||||
d.keepCase = d.flagArg(f)
|
||||
case "NOSUGGEST":
|
||||
d.noSuggest = d.flagArg(f)
|
||||
case "ONLYINCOMPOUND":
|
||||
d.onlyInComp = d.flagArg(f)
|
||||
case "CIRCUMFIX":
|
||||
d.circumfix = d.flagArg(f)
|
||||
case "COMPOUNDFLAG":
|
||||
d.compound.any = d.flagArg(f)
|
||||
case "COMPOUNDBEGIN":
|
||||
d.compound.begin = d.flagArg(f)
|
||||
case "COMPOUNDMIDDLE":
|
||||
d.compound.middle = d.flagArg(f)
|
||||
case "COMPOUNDLAST", "COMPOUNDEND":
|
||||
d.compound.last = d.flagArg(f)
|
||||
case "COMPOUNDPERMITFLAG":
|
||||
d.compound.permit = d.flagArg(f)
|
||||
case "COMPOUNDFORBIDFLAG":
|
||||
d.compound.forbid = d.flagArg(f)
|
||||
case "COMPOUNDMIN":
|
||||
if len(f) >= 2 {
|
||||
if n, err := strconv.Atoi(f[1]); err == nil {
|
||||
d.compound.min = max(n, 1)
|
||||
}
|
||||
}
|
||||
case "COMPOUNDWORDMAX":
|
||||
if len(f) >= 2 {
|
||||
d.compound.maxWords, _ = strconv.Atoi(f[1])
|
||||
}
|
||||
case "CHECKCOMPOUNDDUP":
|
||||
d.compound.checkDup = true
|
||||
case "CHECKCOMPOUNDCASE":
|
||||
d.compound.checkCase = true
|
||||
case "PFX", "SFX":
|
||||
if len(f) < 4 {
|
||||
continue
|
||||
}
|
||||
key := f[0] + " " + f[1]
|
||||
if pending[key] == 0 {
|
||||
n, err := strconv.Atoi(f[3])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
pending[key] = n
|
||||
cross[key] = f[2] == "Y"
|
||||
continue
|
||||
}
|
||||
pending[key]--
|
||||
d.addAffix(f, cross[key])
|
||||
}
|
||||
}
|
||||
for _, p := range pairs(lines, "REP") {
|
||||
r := replacement{from: p[0], to: strings.ReplaceAll(p[1], "_", " ")}
|
||||
r.start = strings.HasPrefix(r.from, "^")
|
||||
r.end = strings.HasSuffix(r.from, "$")
|
||||
r.from = strings.TrimSuffix(strings.TrimPrefix(r.from, "^"), "$")
|
||||
r.from = strings.ReplaceAll(r.from, "_", " ")
|
||||
if r.from != "" {
|
||||
d.rep = append(d.rep, r)
|
||||
}
|
||||
}
|
||||
d.iconv = replacer(pairs(lines, "ICONV"))
|
||||
d.oconv = replacer(pairs(lines, "OCONV"))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) flagArg(f []string) flag {
|
||||
if len(f) < 2 {
|
||||
return 0
|
||||
}
|
||||
return d.oneFlag(f[1])
|
||||
}
|
||||
|
||||
func replacer(table [][2]string) *strings.Replacer {
|
||||
var args []string
|
||||
for _, p := range table {
|
||||
args = append(args, p[0], p[1])
|
||||
}
|
||||
return strings.NewReplacer(args...)
|
||||
}
|
||||
|
||||
// addAffix reads one entry line: "SFX flag strip add[/cont] [condition]".
|
||||
func (d *Dictionary) addAffix(f []string, cross bool) {
|
||||
a := &affix{flag: d.oneFlag(f[1]), cross: cross}
|
||||
if f[2] != "0" {
|
||||
a.strip = f[2]
|
||||
}
|
||||
add := f[3]
|
||||
if i := strings.IndexByte(add, '/'); i >= 0 {
|
||||
a.cont = d.parseFlags(add[i+1:])
|
||||
add = add[:i]
|
||||
}
|
||||
if add != "0" {
|
||||
a.add = add
|
||||
}
|
||||
if len(f) >= 5 {
|
||||
a.cond = parseCondition(f[4])
|
||||
}
|
||||
if f[0] == "PFX" {
|
||||
d.prefixes[a.add] = append(d.prefixes[a.add], a)
|
||||
d.maxPrefix = max(d.maxPrefix, len(a.add))
|
||||
return
|
||||
}
|
||||
d.suffixes[a.add] = append(d.suffixes[a.add], a)
|
||||
d.maxSuffix = max(d.maxSuffix, len(a.add))
|
||||
for _, c := range a.cont {
|
||||
d.contFlags[c] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dictionary) parseDic(text string) {
|
||||
first := true
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
if first {
|
||||
first = false
|
||||
if _, err := strconv.Atoi(strings.TrimSpace(line)); err == nil {
|
||||
continue // approximate word count
|
||||
}
|
||||
}
|
||||
if line == "" || line[0] == '\t' || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
// Morphological fields follow a tab or a space.
|
||||
if i := strings.IndexAny(line, "\t "); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
word, flags := splitEntry(line)
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
var set flagSet
|
||||
if flags != "" {
|
||||
set = d.parseFlags(flags)
|
||||
}
|
||||
d.words[word] = append(d.words[word], set)
|
||||
}
|
||||
}
|
||||
|
||||
// splitEntry separates "word/flags", where "\/" is a literal slash.
|
||||
func splitEntry(s string) (word, flags string) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+1 < len(s) && s[i+1] == '/' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if s[i] == '/' {
|
||||
return strings.ReplaceAll(s[:i], `\/`, "/"), s[i+1:]
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(s, `\/`, "/"), ""
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package spell
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"plotzilla/internal/screenplay"
|
||||
)
|
||||
|
||||
const enAff = `SET UTF-8
|
||||
TRY esianrtolcdugmphbyfvkwz'
|
||||
ICONV 1
|
||||
ICONV ’ '
|
||||
NOSUGGEST !
|
||||
KEEPCASE K
|
||||
FORBIDDENWORD F
|
||||
NEEDAFFIX N
|
||||
REP 1
|
||||
REP alot a_lot
|
||||
|
||||
PFX U Y 1
|
||||
PFX U 0 un .
|
||||
|
||||
SFX S Y 4
|
||||
SFX S y ies [^aeiou]y
|
||||
SFX S 0 s [aeiou]y
|
||||
SFX S 0 s [^sxzy]
|
||||
SFX S 0 es [sxz]
|
||||
|
||||
SFX D Y 2
|
||||
SFX D 0 ed [^e]
|
||||
SFX D 0 d e
|
||||
|
||||
SFX M Y 1
|
||||
SFX M 0 's .
|
||||
|
||||
# "ness" allows a plural on top of it: kindnesses.
|
||||
SFX Q Y 1
|
||||
SFX Q 0 ness/S .
|
||||
`
|
||||
|
||||
const enDic = `16
|
||||
kind/QU
|
||||
open/SD
|
||||
for
|
||||
door/S
|
||||
city/SM
|
||||
play/SD
|
||||
love/DS
|
||||
the
|
||||
a
|
||||
lot
|
||||
Paris/M
|
||||
iPhone
|
||||
OpenGL/K
|
||||
walkd/F
|
||||
realis/N
|
||||
rude/!
|
||||
`
|
||||
|
||||
func testDict(t *testing.T) *Dictionary {
|
||||
t.Helper()
|
||||
d, err := Parse([]byte(enAff), []byte(enDic))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestCheck(t *testing.T) {
|
||||
d := testDict(t)
|
||||
for word, want := range map[string]bool{
|
||||
"door": true,
|
||||
"doors": true,
|
||||
"cities": true,
|
||||
"citys": false, // condition [^y] rejects it
|
||||
"plays": true,
|
||||
"played": true,
|
||||
"loved": true,
|
||||
"loveed": false,
|
||||
"unkind": true,
|
||||
"kindness": true,
|
||||
"kindnesses": true, // two suffixes
|
||||
"unkindness": true, // cross product
|
||||
"undoor": false,
|
||||
"city's": true,
|
||||
"city’s": true, // typographic apostrophe
|
||||
"Door": true,
|
||||
"DOORS": true,
|
||||
"Paris": true,
|
||||
"PARIS": true,
|
||||
"Paris's": true,
|
||||
"paris": false, // proper nouns keep their capital
|
||||
"iPhone": true,
|
||||
"OpenGL": true,
|
||||
"OPENGL": false, // KEEPCASE
|
||||
"walkd": false, // FORBIDDENWORD
|
||||
"realis": false, // NEEDAFFIX
|
||||
"cty": false,
|
||||
"": false,
|
||||
"rude": true,
|
||||
"UnKind": false,
|
||||
"the": true,
|
||||
"supercalifrag": false,
|
||||
} {
|
||||
if got := d.Check(word); got != want {
|
||||
t.Errorf("Check(%q) = %v, want %v", word, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggest(t *testing.T) {
|
||||
d := testDict(t)
|
||||
for word, want := range map[string][]string{
|
||||
"teh": {"the"},
|
||||
"dor": {"for", "door"},
|
||||
"doorz": {"door", "doors"},
|
||||
"DOOR'S": {"DOORS"},
|
||||
"Citys": {"City", "City's"},
|
||||
"paris": {"Paris"},
|
||||
"alot": {"a lot", "lot"},
|
||||
"thedoor": {"the door"},
|
||||
"rudee": nil, // NOSUGGEST
|
||||
"luve": {"love"},
|
||||
} {
|
||||
if got := d.Suggest(word); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Suggest(%q) = %q, want %q", word, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Long flags, flag aliases and prefixes allowed by a suffix's
|
||||
// continuation class, as in the French dictionary ("l'homme", "d'hommes").
|
||||
func TestAliasesAndLongFlags(t *testing.T) {
|
||||
aff := `SET UTF-8
|
||||
FLAG long
|
||||
AF 2
|
||||
AF L'S.
|
||||
AF S.
|
||||
|
||||
PFX L' Y 1
|
||||
PFX L' 0 l' [aeiouh]
|
||||
|
||||
SFX S. Y 1
|
||||
SFX S. 0 s/L' [^sxz]
|
||||
`
|
||||
dic := "2\nhomme/1\nami/2\n"
|
||||
d, err := Parse([]byte(aff), []byte(dic))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for word, want := range map[string]bool{
|
||||
"homme": true, "hommes": true, "l'homme": true, "l'hommes": true,
|
||||
"L'homme": true, "amis": true, "l'ami": false, "l'amis": true,
|
||||
} {
|
||||
if got := d.Check(word); got != want {
|
||||
t.Errorf("Check(%q) = %v, want %v", word, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatin1(t *testing.T) {
|
||||
aff := []byte("SET ISO8859-1\nTRY ae\xe4\n")
|
||||
dic := []byte("1\nH\xe4user\n")
|
||||
d, err := Parse(aff, dic)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Check("Häuser") || d.Check("Hauser") {
|
||||
t.Error("Latin-1 dictionary not decoded")
|
||||
}
|
||||
if got := d.Suggest("Hauser"); !slices.Contains(got, "Häuser") {
|
||||
t.Errorf("Suggest(Hauser) = %q", got)
|
||||
}
|
||||
if _, err := Parse([]byte("SET KOI8-R\n"), nil); err == nil {
|
||||
t.Error("unsupported encoding accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWords(t *testing.T) {
|
||||
got := Words("Élise's well-known R2D2, l’homme… 'quoted' x")
|
||||
want := []Word{
|
||||
{"Élise's", 0, 7}, {"well", 8, 4}, {"known", 13, 5}, {"R2D2", 19, 4},
|
||||
{"l’homme", 25, 7}, {"quoted", 35, 6}, {"x", 43, 1},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Words = %+v\nwant %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMisspellings(t *testing.T) {
|
||||
c := NewChecker(testDict(t), []string{"Zorblax"})
|
||||
doc := &screenplay.Document{Elements: []screenplay.Element{
|
||||
screenplay.NewElement(screenplay.Scene, "INT. THE CTY - DAY"),
|
||||
screenplay.NewElement(screenplay.Action, "GRUNDLE opens the dor for Zorblax."),
|
||||
screenplay.NewElement(screenplay.Character, "Grundle (CONT'D)"),
|
||||
screenplay.NewElement(screenplay.Dialogue, "Élan loves 3rd doors"),
|
||||
screenplay.NewElement(screenplay.Action, "GRUNDLE's door."),
|
||||
}}
|
||||
got := c.Misspellings(doc)
|
||||
want := []Misspelling{
|
||||
{Element: 0, Offset: 9, Length: 3, Word: "CTY"},
|
||||
{Element: 0, Offset: 15, Length: 3, Word: "DAY"},
|
||||
{Element: 1, Offset: 18, Length: 3, Word: "dor"},
|
||||
{Element: 3, Offset: 0, Length: 4, Word: "Élan"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Misspellings = %+v\nwant %+v", got, want)
|
||||
}
|
||||
|
||||
c.Add("élan")
|
||||
c.Add("day")
|
||||
if got := c.Misspellings(doc); len(got) != 2 {
|
||||
t.Errorf("after Add, Misspellings = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAvailable(t *testing.T) {
|
||||
a, b := t.TempDir(), t.TempDir()
|
||||
for _, f := range []string{
|
||||
filepath.Join(a, "fr_FR.aff"), filepath.Join(a, "fr_FR.dic"),
|
||||
filepath.Join(a, "de.dic"), // no .aff: skipped
|
||||
filepath.Join(a, "hyph_fr.dic"), filepath.Join(a, "hyph_fr.aff"),
|
||||
filepath.Join(b, "fr_FR.aff"), filepath.Join(b, "fr_FR.dic"),
|
||||
filepath.Join(b, "en_US.aff"), filepath.Join(b, "en_US.dic"),
|
||||
} {
|
||||
if err := os.WriteFile(f, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got := Available([]string{a, b, filepath.Join(a, "missing")})
|
||||
want := []Language{
|
||||
{"fr_FR", filepath.Join(a, "fr_FR.aff"), filepath.Join(a, "fr_FR.dic")},
|
||||
{"en_US", filepath.Join(b, "en_US.aff"), filepath.Join(b, "en_US.dic")},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Available = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreferred(t *testing.T) {
|
||||
codes := []string{"de_DE_frami", "en_GB", "en_US", "fr"}
|
||||
for locale, want := range map[string]string{
|
||||
"fr_FR.UTF-8": "fr",
|
||||
"en_GB.UTF-8": "en_GB",
|
||||
"de_AT.UTF-8": "de_DE_frami",
|
||||
"C": "en_US",
|
||||
"": "en_US",
|
||||
} {
|
||||
if got := Preferred(codes, locale); got != want {
|
||||
t.Errorf("Preferred(%q) = %q, want %q", locale, got, want)
|
||||
}
|
||||
}
|
||||
if got := Preferred(nil, "fr_FR"); got != "" {
|
||||
t.Errorf("Preferred(nil) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompounds(t *testing.T) {
|
||||
// German-style compounds: parts marked for the start, middle or end,
|
||||
// a linking "s" that only exists inside compounds, and an affix that
|
||||
// lower-cases a noun inside a compound (COMPOUNDPERMITFLAG).
|
||||
aff := `SET UTF-8
|
||||
COMPOUNDBEGIN B
|
||||
COMPOUNDMIDDLE M
|
||||
COMPOUNDLAST E
|
||||
ONLYINCOMPOUND O
|
||||
COMPOUNDPERMITFLAG P
|
||||
COMPOUNDMIN 1
|
||||
COMPOUNDWORDMAX 3
|
||||
CHECKCOMPOUNDDUP
|
||||
|
||||
PFX L Y 3
|
||||
PFX L T t/P T
|
||||
PFX L S s/P S
|
||||
PFX L Z z/P Z
|
||||
|
||||
SFX N Y 1
|
||||
SFX N 0 en .
|
||||
`
|
||||
dic := `6
|
||||
Haus/BM
|
||||
Tür/LMEN
|
||||
Schlüssel/BMLE
|
||||
s/MO
|
||||
Arbeit/BM
|
||||
Zeit/LE
|
||||
`
|
||||
d, err := Parse([]byte(aff), []byte(dic))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, w := range []string{"Haus", "Haustür", "Haustüren", "Haustürschlüssel", "Arbeitszeit", "HAUSTÜR"} {
|
||||
if !d.Check(w) {
|
||||
t.Errorf("%q should be correct", w)
|
||||
}
|
||||
}
|
||||
for _, w := range []string{"s", "Türhaus", "Hausshaus", "Haushaus", "Arbeitsarbeitszeit", "Haustürx"} {
|
||||
if d.Check(w) {
|
||||
t.Errorf("%q should be wrong", w)
|
||||
}
|
||||
}
|
||||
|
||||
// COMPOUNDFLAG: any marked word can go anywhere.
|
||||
d, err = Parse([]byte("SET UTF-8\nCOMPOUNDFLAG X\n"), []byte("2\nfoot/X\nball/X\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Check("football") || !d.Check("ballfoot") || d.Check("footbal") {
|
||||
t.Errorf("COMPOUNDFLAG compounds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompoundRule(t *testing.T) {
|
||||
// en_US style: numbers built from digits, ordinal endings that only
|
||||
// exist in compounds, and a word pattern with long flags.
|
||||
aff := `SET UTF-8
|
||||
ONLYINCOMPOUND c
|
||||
COMPOUNDMIN 1
|
||||
COMPOUNDRULE 2
|
||||
COMPOUNDRULE n*1t
|
||||
COMPOUNDRULE n*mp
|
||||
`
|
||||
dic := `8
|
||||
0/nm
|
||||
1/n1
|
||||
2/nm
|
||||
3/nm
|
||||
st/tc
|
||||
nd/pc
|
||||
th/pc
|
||||
rd/pc
|
||||
`
|
||||
d, err := Parse([]byte(aff), []byte(dic))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, w := range []string{"1st", "21st", "2nd", "33rd", "20th"} {
|
||||
if !d.Check(w) {
|
||||
t.Errorf("%q should be correct", w)
|
||||
}
|
||||
}
|
||||
for _, w := range []string{"st", "1nd", "2st", "st1"} {
|
||||
if d.Check(w) {
|
||||
t.Errorf("%q should be wrong", w)
|
||||
}
|
||||
}
|
||||
|
||||
d, err = Parse([]byte("SET UTF-8\nFLAG long\nCOMPOUNDRULE 1\nCOMPOUNDRULE (aa)(bb)?(cc)\n"),
|
||||
[]byte("3\nsun/aa\nflower/cc\nlit/bb\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Check("sunflower") || !d.Check("sunlitflower") || d.Check("flowersun") || d.Check("sunlit") {
|
||||
t.Errorf("long-flag COMPOUNDRULE")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user