fix: share the spelling dictionaries with the translation lexicon
Every model load read the full French and English Hunspell dictionaries from disk again, even when the spell checker already held the same dictionary in memory. App now caches one Dictionary per language; a Dictionary is read-only once loaded, so the checker and the language detection share it.
This commit is contained in:
1 parent
343836df6b
commit
823c9f2d0f
2 files changed
+47
-22
No files matched your search
@@ -33,10 +33,12 @@ type App struct {
|
|||||||
initialPath string
|
initialPath string
|
||||||
|
|
||||||
// Wails runs each call on its own goroutine: spelling calls share one
|
// Wails runs each call on its own goroutine: spelling calls share one
|
||||||
// loaded dictionary.
|
// loaded dictionary. dicts caches them per language so the translation's
|
||||||
|
// language detection does not read the same files again.
|
||||||
spellMu sync.Mutex
|
spellMu sync.Mutex
|
||||||
checker *spell.Checker
|
checker *spell.Checker
|
||||||
checkerLang string
|
checkerLang string
|
||||||
|
dicts map[string]*spell.Dictionary
|
||||||
|
|
||||||
tr translation
|
tr translation
|
||||||
}
|
}
|
||||||
@@ -294,6 +296,28 @@ func (a *App) SpellLanguages() SpellInfo {
|
|||||||
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dictionaryLocked returns the Hunspell dictionary for a language code
|
||||||
|
// ("en_US"), loading it the first time. spellMu must be held.
|
||||||
|
func (a *App) dictionaryLocked(code string) (*spell.Dictionary, error) {
|
||||||
|
if d, ok := a.dicts[code]; ok {
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
langs := spell.Available(dictionaryDirs())
|
||||||
|
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == code })
|
||||||
|
if i < 0 {
|
||||||
|
return nil, fmt.Errorf("no %s dictionary installed", code)
|
||||||
|
}
|
||||||
|
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("loading the %s dictionary: %w", code, err)
|
||||||
|
}
|
||||||
|
if a.dicts == nil {
|
||||||
|
a.dicts = map[string]*spell.Dictionary{}
|
||||||
|
}
|
||||||
|
a.dicts[code] = d
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
// spellChecker returns the checker for lang, loading its dictionary the
|
// spellChecker returns the checker for lang, loading its dictionary the
|
||||||
// first time it is asked for.
|
// first time it is asked for.
|
||||||
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||||
@@ -302,15 +326,9 @@ func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
|||||||
if a.checker != nil && a.checkerLang == lang {
|
if a.checker != nil && a.checkerLang == lang {
|
||||||
return a.checker, nil
|
return a.checker, nil
|
||||||
}
|
}
|
||||||
langs := spell.Available(dictionaryDirs())
|
dict, err := a.dictionaryLocked(lang)
|
||||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == lang })
|
|
||||||
if i < 0 {
|
|
||||||
return nil, fmt.Errorf("no %s dictionary installed", lang)
|
|
||||||
}
|
|
||||||
l := langs[i]
|
|
||||||
dict, err := spell.Load(l.Aff, l.Dic)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("loading the %s dictionary: %w", lang, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
var personal []string
|
var personal []string
|
||||||
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
||||||
|
|||||||
+20
-13
@@ -276,7 +276,7 @@ func (a *App) translator(ctx context.Context) (*marian.Model, translate.Lexicon,
|
|||||||
// Every sentence worker gets an equal share of the CPUs for its
|
// Every sentence worker gets an equal share of the CPUs for its
|
||||||
// decoding steps, so workers × threads never oversubscribe the machine.
|
// decoding steps, so workers × threads never oversubscribe the machine.
|
||||||
m.SetThreads(max(1, runtime.NumCPU()/translateWorkers()))
|
m.SetThreads(max(1, runtime.NumCPU()/translateWorkers()))
|
||||||
lexicon = loadLexicon()
|
lexicon = a.loadLexicon()
|
||||||
a.tr.mu.Lock()
|
a.tr.mu.Lock()
|
||||||
if ctx.Err() == nil { // cancelled while loading: cache nothing
|
if ctx.Err() == nil { // cancelled while loading: cache nothing
|
||||||
a.tr.model, a.tr.lexicon = m, lexicon
|
a.tr.model, a.tr.lexicon = m, lexicon
|
||||||
@@ -316,25 +316,32 @@ func (l lexicon) Knows(word string) (french, english bool) { return l.fr.Check(w
|
|||||||
|
|
||||||
// loadLexicon needs both a French and an English dictionary: with one
|
// loadLexicon needs both a French and an English dictionary: with one
|
||||||
// only, every word it knows would count for its language. Without them,
|
// only, every word it knows would count for its language. Without them,
|
||||||
// detection relies on function words and accents.
|
// detection relies on function words and accents. The dictionaries come
|
||||||
func loadLexicon() translate.Lexicon {
|
// from the same cache as the spell checker's: a Dictionary is read-only
|
||||||
|
// once loaded, so it can be shared.
|
||||||
|
func (a *App) loadLexicon() translate.Lexicon {
|
||||||
langs := spell.Available(dictionaryDirs())
|
langs := spell.Available(dictionaryDirs())
|
||||||
load := func(lang, preferred string) *spell.Dictionary {
|
pick := func(lang, preferred string) string {
|
||||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred })
|
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred }); i >= 0 {
|
||||||
if i < 0 {
|
return langs[i].Code
|
||||||
i = slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") })
|
|
||||||
}
|
}
|
||||||
if i < 0 {
|
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") }); i >= 0 {
|
||||||
|
return langs[i].Code
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
frCode, enCode := pick("fr", "fr_FR"), pick("en", "en_US")
|
||||||
|
if frCode == "" || enCode == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
a.spellMu.Lock()
|
||||||
|
defer a.spellMu.Unlock()
|
||||||
|
fr, err := a.dictionaryLocked(frCode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return d
|
en, err := a.dictionaryLocked(enCode)
|
||||||
}
|
if err != nil {
|
||||||
fr, en := load("fr", "fr_FR"), load("en", "en_US")
|
|
||||||
if fr == nil || en == nil {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return lexicon{fr, en}
|
return lexicon{fr, en}
|
||||||
|
|||||||
Reference in new issue
Block a user