fix: share the spelling dictionaries with the translation lexicon
Every model load read the full French and English Hunspell dictionaries from disk again, even when the spell checker already held the same dictionary in memory. App now caches one Dictionary per language; a Dictionary is read-only once loaded, so the checker and the language detection share it.
This commit is contained in:
1 parent
343836df6b
commit
823c9f2d0f
2 files changed
+47
-22
No files matched your search
@@ -33,10 +33,12 @@ type App struct {
|
||||
initialPath string
|
||||
|
||||
// Wails runs each call on its own goroutine: spelling calls share one
|
||||
// loaded dictionary.
|
||||
// loaded dictionary. dicts caches them per language so the translation's
|
||||
// language detection does not read the same files again.
|
||||
spellMu sync.Mutex
|
||||
checker *spell.Checker
|
||||
checkerLang string
|
||||
dicts map[string]*spell.Dictionary
|
||||
|
||||
tr translation
|
||||
}
|
||||
@@ -294,6 +296,28 @@ func (a *App) SpellLanguages() SpellInfo {
|
||||
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
||||
}
|
||||
|
||||
// dictionaryLocked returns the Hunspell dictionary for a language code
|
||||
// ("en_US"), loading it the first time. spellMu must be held.
|
||||
func (a *App) dictionaryLocked(code string) (*spell.Dictionary, error) {
|
||||
if d, ok := a.dicts[code]; ok {
|
||||
return d, nil
|
||||
}
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == code })
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("no %s dictionary installed", code)
|
||||
}
|
||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading the %s dictionary: %w", code, err)
|
||||
}
|
||||
if a.dicts == nil {
|
||||
a.dicts = map[string]*spell.Dictionary{}
|
||||
}
|
||||
a.dicts[code] = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// spellChecker returns the checker for lang, loading its dictionary the
|
||||
// first time it is asked for.
|
||||
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||
@@ -302,15 +326,9 @@ func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||
if a.checker != nil && a.checkerLang == lang {
|
||||
return a.checker, nil
|
||||
}
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == lang })
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("no %s dictionary installed", lang)
|
||||
}
|
||||
l := langs[i]
|
||||
dict, err := spell.Load(l.Aff, l.Dic)
|
||||
dict, err := a.dictionaryLocked(lang)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading the %s dictionary: %w", lang, err)
|
||||
return nil, err
|
||||
}
|
||||
var personal []string
|
||||
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
||||
|
||||
+20
-13
@@ -276,7 +276,7 @@ func (a *App) translator(ctx context.Context) (*marian.Model, translate.Lexicon,
|
||||
// Every sentence worker gets an equal share of the CPUs for its
|
||||
// decoding steps, so workers × threads never oversubscribe the machine.
|
||||
m.SetThreads(max(1, runtime.NumCPU()/translateWorkers()))
|
||||
lexicon = loadLexicon()
|
||||
lexicon = a.loadLexicon()
|
||||
a.tr.mu.Lock()
|
||||
if ctx.Err() == nil { // cancelled while loading: cache nothing
|
||||
a.tr.model, a.tr.lexicon = m, lexicon
|
||||
@@ -316,25 +316,32 @@ func (l lexicon) Knows(word string) (french, english bool) { return l.fr.Check(w
|
||||
|
||||
// loadLexicon needs both a French and an English dictionary: with one
|
||||
// only, every word it knows would count for its language. Without them,
|
||||
// detection relies on function words and accents.
|
||||
func loadLexicon() translate.Lexicon {
|
||||
// detection relies on function words and accents. The dictionaries come
|
||||
// from the same cache as the spell checker's: a Dictionary is read-only
|
||||
// once loaded, so it can be shared.
|
||||
func (a *App) loadLexicon() translate.Lexicon {
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
load := func(lang, preferred string) *spell.Dictionary {
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred })
|
||||
if i < 0 {
|
||||
i = slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") })
|
||||
pick := func(lang, preferred string) string {
|
||||
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred }); i >= 0 {
|
||||
return langs[i].Code
|
||||
}
|
||||
if i < 0 {
|
||||
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") }); i >= 0 {
|
||||
return langs[i].Code
|
||||
}
|
||||
return ""
|
||||
}
|
||||
frCode, enCode := pick("fr", "fr_FR"), pick("en", "en_US")
|
||||
if frCode == "" || enCode == "" {
|
||||
return nil
|
||||
}
|
||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||
a.spellMu.Lock()
|
||||
defer a.spellMu.Unlock()
|
||||
fr, err := a.dictionaryLocked(frCode)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return d
|
||||
}
|
||||
fr, en := load("fr", "fr_FR"), load("en", "en_US")
|
||||
if fr == nil || en == nil {
|
||||
en, err := a.dictionaryLocked(enCode)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return lexicon{fr, en}
|
||||
|
||||
Reference in new issue
Block a user