Compare commits
17
Commits
839e9291d6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d355d93126 | ||
|
|
2de0d59641 | ||
|
|
093f7ea10d | ||
|
|
764c9d4dc3 | ||
|
|
6622203506 | ||
|
|
b0bcb812b1 | ||
|
|
823c9f2d0f | ||
|
|
343836df6b | ||
|
|
e85d0ba7e9 | ||
|
|
1440adc199 | ||
|
|
82bfef8cba | ||
|
|
d6ee57130f | ||
|
|
e80ad634ef | ||
|
|
5e7ae97b57 | ||
|
|
c0a52a9040 | ||
|
|
643adffb12 | ||
|
|
b6b58c7aca |
No files matched your search
@@ -62,7 +62,8 @@ internal/layout line wrapping + pagination (paper sizes, locked pages);
|
||||
internal/pdf hand-written PDF 1.4 writer, standard Courier fonts, zero dependencies
|
||||
internal/spell Hunspell .aff/.dic reader, checker (compounds too), suggestions, word scan
|
||||
internal/marian MarianMT (OPUS-MT) inference in pure Go: safetensors, SentencePiece unigram,
|
||||
transformer encoder-decoder, beam search, verified download
|
||||
transformer encoder-decoder, beam search, verified download. In-tree on
|
||||
purpose (one binary, no cgo, offline by design): see the package doc.
|
||||
internal/translate French → English script translation: language detection, screenplay term
|
||||
tables, per-element rules, name protection, parallel sentence translation
|
||||
frontend/index.html app shell: header bar, navigator, paper, HUD, <dialog>s
|
||||
@@ -81,7 +82,7 @@ frontend/src/icons.ts inline Lucide-style SVG icons
|
||||
frontend/src/style.css the whole UI theme (dark, blue) and the on-screen page geometry
|
||||
screenshots/ README images (2880×1800 UI captures + pdf-output.png)
|
||||
tools/screenshots/ `make screenshots`: sample script, mock server (main.go), Firefox capture (capture.mjs)
|
||||
VERSION current version (1.0)
|
||||
VERSION current version
|
||||
CHANGELOG.md Keep a Changelog; sections Added, Changed, Fixed only
|
||||
```
|
||||
|
||||
|
||||
@@ -7,6 +7,34 @@ version is in the `VERSION` file.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.4.1] - 2026-10-15
|
||||
|
||||
### Added
|
||||
|
||||
- The PDF export warns when the script contains characters the standard PDF fonts
|
||||
cannot draw: they are written as `?` in the file, and are now listed in a dialog
|
||||
instead of being silently lost.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `plotzilla new-script.fountain` opens a new script that will be saved to that file,
|
||||
instead of an untitled document that forgot the name it was given.
|
||||
- A Final Draft file named `.txt` (or anything but `.fdx`) is recognised by its
|
||||
content and opened as FDX, not parsed as Fountain.
|
||||
- The unsaved-changes marker is now updated atomically: an edit landing while a save
|
||||
finished could make the close confirmation unreliable.
|
||||
- Saving is now durable: the file is synced to disk before it replaces the old one (a
|
||||
power cut just after a save could leave an empty script). An existing file keeps its
|
||||
permissions, and saving through a symlink replaces the file it points to instead of
|
||||
replacing the link.
|
||||
- Translation is faster and uses much less memory: the beam search no longer copies
|
||||
every beam's decoder state for every candidate at every step, and reuses its
|
||||
work buffers instead of allocating new ones per layer and step. Each sentence
|
||||
now uses its share of the processors, so translating a script no longer starts
|
||||
more workers than the machine has.
|
||||
|
||||
## [1.4] - 2026-10-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- The Cancel button responds while the translation model is loading.
|
||||
|
||||
@@ -473,7 +473,8 @@ The title page and first pages of a sample script, as exported:
|
||||

|
||||
|
||||
The standard Courier fonts cover Western European characters (accents, curly quotes, dashes).
|
||||
Other characters, such as Greek, Cyrillic or Asian scripts, are printed as `?`.
|
||||
Other characters, such as Greek, Cyrillic or Asian scripts, are printed as `?`: the export then
|
||||
warns and lists them, so they are not lost silently.
|
||||
|
||||
**Final Draft** (`Ctrl+Shift+E`): writes an `.fdx` file that opens in Final Draft and other apps
|
||||
that read FDX.
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
@@ -24,15 +25,20 @@ import (
|
||||
|
||||
// App is bound to the frontend: every exported method is callable from JS.
|
||||
type App struct {
|
||||
ctx context.Context
|
||||
dirty bool
|
||||
ctx context.Context
|
||||
// dirty decides whether closing the window asks for confirmation. It is
|
||||
// written from Wails call goroutines (SetDirty) and read by the GTK
|
||||
// close handler: atomic, like spellMu protects the checker.
|
||||
dirty atomic.Bool
|
||||
initialPath string
|
||||
|
||||
// Wails runs each call on its own goroutine: spelling calls share one
|
||||
// loaded dictionary.
|
||||
// loaded dictionary. dicts caches them per language so the translation's
|
||||
// language detection does not read the same files again.
|
||||
spellMu sync.Mutex
|
||||
checker *spell.Checker
|
||||
checkerLang string
|
||||
dicts map[string]*spell.Dictionary
|
||||
|
||||
tr translation
|
||||
}
|
||||
@@ -74,7 +80,7 @@ func (a *App) startup(ctx context.Context) {
|
||||
// beforeClose keeps the window open while there are unsaved changes and
|
||||
// lets the frontend ask; it quits again once the user has decided.
|
||||
func (a *App) beforeClose(ctx context.Context) bool {
|
||||
if !a.dirty {
|
||||
if !a.dirty.Load() {
|
||||
return false
|
||||
}
|
||||
runtime.EventsEmit(a.ctx, "confirm-close")
|
||||
@@ -83,13 +89,18 @@ func (a *App) beforeClose(ctx context.Context) bool {
|
||||
|
||||
func (a *App) NewDocument() *screenplay.Document { return screenplay.New() }
|
||||
|
||||
// InitialFile returns the script passed on the command line, if any.
|
||||
// InitialFile returns the script passed on the command line, if any. A
|
||||
// path that does not exist yet opens a new script, written there on the
|
||||
// first save: `plotzilla new-script.fountain` creates the file.
|
||||
func (a *App) InitialFile() (*OpenResult, error) {
|
||||
if a.initialPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
path := a.initialPath
|
||||
a.initialPath = ""
|
||||
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||
return &OpenResult{Path: path, Document: screenplay.New()}, nil
|
||||
}
|
||||
return a.load(path)
|
||||
}
|
||||
|
||||
@@ -107,7 +118,7 @@ func (a *App) load(path string) (*OpenResult, error) {
|
||||
return nil, err
|
||||
}
|
||||
var doc *screenplay.Document
|
||||
if strings.EqualFold(filepath.Ext(path), ".fdx") {
|
||||
if isFDX(path, data) {
|
||||
if doc, err = fdx.Read(bytes.NewReader(data)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -117,6 +128,16 @@ func (a *App) load(path string) (*OpenResult, error) {
|
||||
return &OpenResult{Path: path, Document: doc}, nil
|
||||
}
|
||||
|
||||
// isFDX reports whether a file is a Final Draft one: the extension says
|
||||
// so, or its content does — a ".txt" export of Final Draft is still FDX.
|
||||
func isFDX(path string, data []byte) bool {
|
||||
if strings.EqualFold(filepath.Ext(path), ".fdx") {
|
||||
return true
|
||||
}
|
||||
head := strings.TrimPrefix(string(data[:min(len(data), 512)]), "\ufeff")
|
||||
return strings.Contains(head, "<FinalDraft")
|
||||
}
|
||||
|
||||
// Save writes the document to path, asking for one when empty. It returns
|
||||
// the path written, or "" if the user cancelled.
|
||||
func (a *App) Save(doc screenplay.Document, path string) (string, error) {
|
||||
@@ -147,11 +168,19 @@ func (a *App) SaveAs(doc screenplay.Document, current string) (string, error) {
|
||||
return a.Save(doc, path)
|
||||
}
|
||||
|
||||
func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current string) (string, error) {
|
||||
// ExportResult describes a written PDF export.
|
||||
type ExportResult struct {
|
||||
Path string `json:"path"` // "" when the user cancelled the dialog
|
||||
// Replaced are the characters the standard PDF fonts cannot draw:
|
||||
// each one is written as "?" in the file.
|
||||
Replaced []string `json:"replaced,omitempty"`
|
||||
}
|
||||
|
||||
func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current string) (ExportResult, error) {
|
||||
path, err := a.saveDialog("Export PDF", current, ".pdf",
|
||||
[]runtime.FileFilter{{DisplayName: "PDF (*.pdf)", Pattern: "*.pdf"}})
|
||||
if err != nil || path == "" {
|
||||
return "", err
|
||||
return ExportResult{}, err
|
||||
}
|
||||
var pages []layout.Page
|
||||
if opts.TitlePage && !doc.TitlePage.IsEmpty() {
|
||||
@@ -163,11 +192,18 @@ func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current str
|
||||
pdfOpts.Revisions = doc.Settings.Revisions
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = pdf.Render(&buf, pages, pdfOpts)
|
||||
replaced, err := pdf.Render(&buf, pages, pdfOpts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return ExportResult{}, err
|
||||
}
|
||||
return path, writeAtomic(path, buf.Bytes())
|
||||
if err := writeAtomic(path, buf.Bytes()); err != nil {
|
||||
return ExportResult{}, err
|
||||
}
|
||||
res := ExportResult{Path: path}
|
||||
for _, r := range replaced {
|
||||
res.Replaced = append(res.Replaced, string(r))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (a *App) ExportFDX(doc screenplay.Document, current string) (string, error) {
|
||||
@@ -201,7 +237,7 @@ func (a *App) RevisionSet(n int) screenplay.Revision {
|
||||
return r
|
||||
}
|
||||
|
||||
func (a *App) SetDirty(dirty bool) { a.dirty = dirty }
|
||||
func (a *App) SetDirty(dirty bool) { a.dirty.Store(dirty) }
|
||||
|
||||
// dataDir is the XDG data directory (~/.local/share).
|
||||
func dataDir() string {
|
||||
@@ -275,6 +311,28 @@ func (a *App) SpellLanguages() SpellInfo {
|
||||
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
||||
}
|
||||
|
||||
// dictionaryLocked returns the Hunspell dictionary for a language code
|
||||
// ("en_US"), loading it the first time. spellMu must be held.
|
||||
func (a *App) dictionaryLocked(code string) (*spell.Dictionary, error) {
|
||||
if d, ok := a.dicts[code]; ok {
|
||||
return d, nil
|
||||
}
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == code })
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("no %s dictionary installed", code)
|
||||
}
|
||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading the %s dictionary: %w", code, err)
|
||||
}
|
||||
if a.dicts == nil {
|
||||
a.dicts = map[string]*spell.Dictionary{}
|
||||
}
|
||||
a.dicts[code] = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// spellChecker returns the checker for lang, loading its dictionary the
|
||||
// first time it is asked for.
|
||||
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||
@@ -283,15 +341,9 @@ func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||
if a.checker != nil && a.checkerLang == lang {
|
||||
return a.checker, nil
|
||||
}
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == lang })
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("no %s dictionary installed", lang)
|
||||
}
|
||||
l := langs[i]
|
||||
dict, err := spell.Load(l.Aff, l.Dic)
|
||||
dict, err := a.dictionaryLocked(lang)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading the %s dictionary: %w", lang, err)
|
||||
return nil, err
|
||||
}
|
||||
var personal []string
|
||||
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
||||
@@ -382,20 +434,48 @@ func (a *App) saveDialog(title, current, ext string, filters []runtime.FileFilte
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// writeAtomic writes data to path atomically and durably: a temporary file
|
||||
// in the same directory is written, given the file's permissions and synced
|
||||
// to disk, then renamed over path and the directory itself synced, so a
|
||||
// power cut just after a save cannot leave an empty script behind (the very
|
||||
// failure autosave, which calls this every 1.2 s, exists to prevent). A
|
||||
// symlinked path is resolved first: the file it points to is replaced, not
|
||||
// the link. An existing file keeps its permissions; a new one gets 0o644.
|
||||
func writeAtomic(path string, data []byte) error {
|
||||
if target, err := filepath.EvalSymlinks(path); err == nil {
|
||||
path = target
|
||||
}
|
||||
mode := os.FileMode(0o644)
|
||||
if st, err := os.Stat(path); err == nil {
|
||||
mode = st.Mode().Perm()
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, werr := tmp.Write(data)
|
||||
cerr := tmp.Close()
|
||||
if err := errors.Join(werr, cerr); err != nil {
|
||||
if _, err = tmp.Write(data); err == nil {
|
||||
err = tmp.Chmod(mode)
|
||||
}
|
||||
if err == nil {
|
||||
err = tmp.Sync()
|
||||
}
|
||||
if cerr := tmp.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
// The rename is a directory entry: sync the directory so it too
|
||||
// survives a crash.
|
||||
dir, err := os.Open(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
Generated
+3
-3
@@ -1155,9 +1155,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz",
|
||||
"integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
|
||||
@@ -144,13 +144,21 @@ export interface ExportOptions {
|
||||
revisions: boolean
|
||||
}
|
||||
|
||||
/** A written PDF export. */
|
||||
export interface ExportResult {
|
||||
/** Empty when the user cancelled the dialog. */
|
||||
path: string
|
||||
/** Characters the standard PDF fonts cannot draw, written as "?" in the file. */
|
||||
replaced?: string[]
|
||||
}
|
||||
|
||||
interface GoApp {
|
||||
NewDocument(): Promise<ScriptDocument>
|
||||
InitialFile(): Promise<OpenResult | null>
|
||||
Open(): Promise<OpenResult | null>
|
||||
Save(doc: ScriptDocument, path: string): Promise<string>
|
||||
SaveAs(doc: ScriptDocument, current: string): Promise<string>
|
||||
ExportPDF(doc: ScriptDocument, opts: ExportOptions, current: string): Promise<string>
|
||||
ExportPDF(doc: ScriptDocument, opts: ExportOptions, current: string): Promise<ExportResult>
|
||||
ExportFDX(doc: ScriptDocument, current: string): Promise<string>
|
||||
Paginate(doc: ScriptDocument): Promise<LayoutInfo>
|
||||
ParseFountain(text: string): Promise<Element[] | null>
|
||||
|
||||
@@ -947,7 +947,15 @@ const actions: Record<string, () => unknown> = {
|
||||
renderPaper()
|
||||
settingsChanged()
|
||||
}
|
||||
await api.ExportPDF(script(), chosen, app.path).catch(reportError)
|
||||
const res = await api.ExportPDF(script(), chosen, app.path).catch(reportError)
|
||||
if (res?.replaced?.length) {
|
||||
const list = res.replaced.map((c) => `"${c}"`).join(' ')
|
||||
await messageBox({
|
||||
title: 'Characters replaced in the PDF',
|
||||
text: `The standard PDF fonts cannot draw these characters: ${list}. Each one is written as "?" in the exported file.`,
|
||||
ok: 'OK',
|
||||
})
|
||||
}
|
||||
}
|
||||
dialog.showModal()
|
||||
},
|
||||
|
||||
+64
-27
@@ -13,7 +13,9 @@ type beam struct {
|
||||
tokens []int // starts with the decoder start token
|
||||
score float64
|
||||
// Keys and values of the decoder's self-attention, one slice per layer
|
||||
// holding len(tokens) rows: each step only computes its own row.
|
||||
// holding len(tokens) rows: each step only computes its own row. When a
|
||||
// beam is the only survivor of its parent it takes the parent's slices
|
||||
// over instead of copying them (the parent is dead from that step on).
|
||||
keys, values [][]float32
|
||||
}
|
||||
|
||||
@@ -27,6 +29,16 @@ type candidate struct {
|
||||
score float64
|
||||
}
|
||||
|
||||
// decoderScratch holds the work buffers of a decoding step, shared by the
|
||||
// layers, the beams and the steps of a search: allocating them afresh for
|
||||
// every layer of every step costs more than the dot products they serve.
|
||||
type decoderScratch struct {
|
||||
q, k, v []float32
|
||||
ctxv, tmp []float32
|
||||
scores []float32
|
||||
hidden []float32
|
||||
}
|
||||
|
||||
// beamSearch generates the translation, as Hugging Face's generate() does
|
||||
// with the model's defaults: num_beams beams, length penalty 1, the
|
||||
// padding token never produced.
|
||||
@@ -35,18 +47,32 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
||||
numBeams := max(m.cfg.NumBeams, 1)
|
||||
crossK := make([][]float32, len(m.decoder))
|
||||
crossV := make([][]float32, len(m.decoder))
|
||||
ffn := 0
|
||||
for i, l := range m.decoder {
|
||||
crossK[i] = make([]float32, srcLen*d)
|
||||
crossV[i] = make([]float32, srcLen*d)
|
||||
l.cross.k.rows(crossK[i], enc, srcLen)
|
||||
l.cross.v.rows(crossV[i], enc, srcLen)
|
||||
ffn = max(ffn, l.fc1.out)
|
||||
}
|
||||
maxLen := min(m.cfg.MaxPositions-1, 2*srcLen+10)
|
||||
scratch := &decoderScratch{
|
||||
q: make([]float32, d), k: make([]float32, d), v: make([]float32, d),
|
||||
ctxv: make([]float32, d), tmp: make([]float32, d),
|
||||
scores: make([]float32, max(maxLen+1, srcLen)),
|
||||
hidden: make([]float32, ffn),
|
||||
}
|
||||
// One vocabulary of logits per beam, reused at every step: the largest
|
||||
// allocation of the search.
|
||||
probs := make([][]float32, numBeams)
|
||||
for i := range probs {
|
||||
probs[i] = make([]float32, m.cfg.VocabSize)
|
||||
}
|
||||
beams := []beam{{
|
||||
tokens: []int{m.cfg.DecoderStart},
|
||||
keys: make([][]float32, len(m.decoder)),
|
||||
values: make([][]float32, len(m.decoder)),
|
||||
}}
|
||||
maxLen := min(m.cfg.MaxPositions-1, 2*srcLen+10)
|
||||
var done []hypothesis
|
||||
for len(beams[0].tokens) <= maxLen {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -54,9 +80,9 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
||||
}
|
||||
hidden := make([][]float32, len(beams))
|
||||
for i := range beams {
|
||||
hidden[i] = m.decodeStep(&beams[i], crossK, crossV, srcLen)
|
||||
hidden[i] = m.decodeStep(&beams[i], crossK, crossV, srcLen, scratch)
|
||||
}
|
||||
logProbs := m.logProbs(hidden)
|
||||
logProbs := m.logProbs(hidden, probs[:len(beams)])
|
||||
var cands []candidate
|
||||
for i, lp := range logProbs {
|
||||
for _, t := range topK(lp, 2*numBeams) {
|
||||
@@ -72,29 +98,45 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
||||
}
|
||||
return 0
|
||||
})
|
||||
var next []beam
|
||||
// Choose the surviving beams first, then build them: a cache is
|
||||
// copied at most once per step, and only when several survivors
|
||||
// share its parent.
|
||||
children := make([]int, len(beams))
|
||||
chosen := make([]candidate, 0, numBeams)
|
||||
for rank, c := range cands {
|
||||
parent := beams[c.parent]
|
||||
if c.token == m.cfg.EOSToken {
|
||||
if rank < numBeams {
|
||||
parent := beams[c.parent]
|
||||
done = append(done, hypothesis{tokens: slices.Clone(parent.tokens), score: c.score / float64(len(parent.tokens))})
|
||||
}
|
||||
continue
|
||||
}
|
||||
chosen = append(chosen, c)
|
||||
children[c.parent]++
|
||||
if len(chosen) == numBeams {
|
||||
break
|
||||
}
|
||||
}
|
||||
next := make([]beam, 0, len(chosen))
|
||||
for _, c := range chosen {
|
||||
parent := &beams[c.parent]
|
||||
child := beam{
|
||||
tokens: append(slices.Clone(parent.tokens), c.token),
|
||||
score: c.score,
|
||||
keys: make([][]float32, len(parent.keys)),
|
||||
values: make([][]float32, len(parent.values)),
|
||||
}
|
||||
takeOver := children[c.parent] == 1
|
||||
for l := range parent.keys {
|
||||
child.keys[l] = slices.Clone(parent.keys[l])
|
||||
child.values[l] = slices.Clone(parent.values[l])
|
||||
if takeOver {
|
||||
child.keys[l] = parent.keys[l]
|
||||
child.values[l] = parent.values[l]
|
||||
} else {
|
||||
child.keys[l] = slices.Clone(parent.keys[l])
|
||||
child.values[l] = slices.Clone(parent.values[l])
|
||||
}
|
||||
}
|
||||
next = append(next, child)
|
||||
if len(next) == numBeams {
|
||||
break
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(done, func(a, b hypothesis) int {
|
||||
switch {
|
||||
@@ -126,22 +168,17 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
||||
|
||||
// decodeStep runs the decoder for the beam's last token and returns its
|
||||
// output vector.
|
||||
func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []float32 {
|
||||
func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int, s *decoderScratch) []float32 {
|
||||
d := m.cfg.DModel
|
||||
pos := len(b.tokens) - 1
|
||||
tok := b.tokens[pos]
|
||||
x := make([]float32, d)
|
||||
x := make([]float32, d) // the beam's output vector, which lives past the step
|
||||
emb := m.embed[tok*d : (tok+1)*d]
|
||||
p := m.decPos[pos*d : (pos+1)*d]
|
||||
for j := range x {
|
||||
x[j] = emb[j]*m.embedScale + p[j]
|
||||
}
|
||||
q := make([]float32, d)
|
||||
k := make([]float32, d)
|
||||
v := make([]float32, d)
|
||||
ctxv := make([]float32, d)
|
||||
tmp := make([]float32, d)
|
||||
scores := make([]float32, max(pos+1, srcLen))
|
||||
q, k, v, ctxv, tmp, scores := s.q, s.k, s.v, s.ctxv, s.tmp, s.scores
|
||||
for i, l := range m.decoder {
|
||||
l.self.q.apply(q, x)
|
||||
l.self.k.apply(k, x)
|
||||
@@ -157,7 +194,7 @@ func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []fl
|
||||
l.cross.o.apply(tmp, ctxv)
|
||||
addNorm(x, tmp, l.crossNorm, 1)
|
||||
|
||||
hidden := make([]float32, l.fc1.out)
|
||||
hidden := s.hidden[:l.fc1.out]
|
||||
l.fc1.apply(hidden, x)
|
||||
for j := range hidden {
|
||||
hidden[j] = m.activation(hidden[j])
|
||||
@@ -169,15 +206,15 @@ func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []fl
|
||||
}
|
||||
|
||||
// logProbs projects each beam's vector on the vocabulary (the embedding
|
||||
// matrix, shared with the output layer) and returns log-softmax scores.
|
||||
// This is the largest product of a step, so it is split across CPUs.
|
||||
func (m *Model) logProbs(hidden [][]float32) [][]float32 {
|
||||
// matrix, shared with the output layer) and returns log-softmax scores in
|
||||
// out, one vocabulary per beam, reused from step to step. This is the
|
||||
// largest product of a step, so it is split across CPUs.
|
||||
func (m *Model) logProbs(hidden, out [][]float32) [][]float32 {
|
||||
d, vocab := m.cfg.DModel, m.cfg.VocabSize
|
||||
out := make([][]float32, len(hidden))
|
||||
for i := range out {
|
||||
out[i] = make([]float32, vocab)
|
||||
}
|
||||
workers := runtime.GOMAXPROCS(0)
|
||||
if m.threads > 0 {
|
||||
workers = min(workers, m.threads)
|
||||
}
|
||||
chunk := (vocab + workers - 1) / workers
|
||||
var wg sync.WaitGroup
|
||||
for lo := 0; lo < vocab; lo += chunk {
|
||||
|
||||
@@ -21,16 +21,6 @@ type RemoteFile struct {
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// Installed reports whether dir holds every file of a model.
|
||||
func Installed(dir string) bool {
|
||||
for _, f := range Files {
|
||||
if st, err := os.Stat(filepath.Join(dir, f)); err != nil || st.Size() == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Download fetches files into dir. A file already there with the right
|
||||
// size and digest is skipped. A file is downloaded into a ".partial" file
|
||||
// that is renamed only once its digest checks out, so an interrupted
|
||||
|
||||
@@ -251,9 +251,6 @@ func tinyModel(t *testing.T) string {
|
||||
|
||||
func TestTinyModel(t *testing.T) {
|
||||
dir := tinyModel(t)
|
||||
if !Installed(dir) {
|
||||
t.Fatal("tiny model not installed")
|
||||
}
|
||||
m, err := Load(context.Background(), dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -281,9 +278,6 @@ func TestTinyModel(t *testing.T) {
|
||||
}
|
||||
|
||||
os.Remove(filepath.Join(dir, "vocab.json"))
|
||||
if Installed(dir) {
|
||||
t.Error("incomplete model reported installed")
|
||||
}
|
||||
if _, err := Load(context.Background(), dir); err == nil {
|
||||
t.Error("incomplete model loaded")
|
||||
}
|
||||
|
||||
+33
-10
@@ -3,6 +3,18 @@
|
||||
// plain Go: a SentencePiece tokenizer, the transformer encoder-decoder and
|
||||
// beam search. A model directory holds config.json, model.safetensors,
|
||||
// source.spm, target.spm and vocab.json.
|
||||
//
|
||||
// The engine lives in-tree on purpose. Plotzilla ships as one binary built
|
||||
// with only the Go toolchain: a native runtime (ctranslate2, argos) would
|
||||
// add cgo and per-distribution packaging, and a separate binary or plugin
|
||||
// an install step, all for a single menu item. Translation is offline by
|
||||
// design — an unpublished script never leaves the machine — and the model
|
||||
// is data downloaded at runtime (Apache-2.0), so neither the "no
|
||||
// third-party dependency" rule nor the GPL is affected. The screenplay
|
||||
// conventions stay out of this package (internal/translate owns them) and
|
||||
// the pipeline is covered by a synthetic tiny model and, on demand, the
|
||||
// real one. If more language pairs are ever added, each with its own
|
||||
// 300 MB download and quirks, reconsider a separate optional component.
|
||||
package marian
|
||||
|
||||
import (
|
||||
@@ -14,9 +26,6 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Files are the files a model directory must contain.
|
||||
var Files = []string{"config.json", "model.safetensors", "source.spm", "target.spm", "vocab.json"}
|
||||
|
||||
type config struct {
|
||||
DModel int `json:"d_model"`
|
||||
EncoderLayers int `json:"encoder_layers"`
|
||||
@@ -60,10 +69,12 @@ type decoderLayer struct {
|
||||
finalLayerNorm layerNorm
|
||||
}
|
||||
|
||||
// Model is a loaded translation model. It is read-only once loaded:
|
||||
// Translate may be called from several goroutines at once.
|
||||
// Model is a loaded translation model. It is read-only once loaded (but
|
||||
// for SetThreads, right after Load): Translate may be called from several
|
||||
// goroutines at once.
|
||||
type Model struct {
|
||||
cfg config
|
||||
threads int // goroutines a decoding step may start (0: one per CPU)
|
||||
embed []float32 // vocab × d, shared by encoder, decoder and output
|
||||
encPos []float32 // positions × d
|
||||
decPos []float32
|
||||
@@ -221,6 +232,13 @@ func (m *Model) bind(t map[string]tensor) error {
|
||||
return missing
|
||||
}
|
||||
|
||||
// SetThreads caps how many goroutines each decoding step starts (0: one
|
||||
// per CPU, the default). When several sentences are translated at once,
|
||||
// give each its share of the CPUs, or every step of every sentence starts
|
||||
// a team of goroutines and they oversubscribe the machine. It must be
|
||||
// called before the first Translate.
|
||||
func (m *Model) SetThreads(n int) { m.threads = n }
|
||||
|
||||
// maxSourceTokens leaves room for the end-of-sentence token.
|
||||
func (m *Model) maxSourceTokens() int { return m.cfg.MaxPositions - 1 }
|
||||
|
||||
@@ -276,6 +294,11 @@ func (m *Model) encode(ids []int) []float32 {
|
||||
ctxv := make([]float32, n*d)
|
||||
tmp := make([]float32, n*d)
|
||||
scores := make([]float32, n)
|
||||
ffn := 0 // the hidden layer of every encoder layer, allocated once
|
||||
for _, l := range m.encoder {
|
||||
ffn = max(ffn, l.fc1.out)
|
||||
}
|
||||
hidden := make([]float32, n*ffn)
|
||||
for _, l := range m.encoder {
|
||||
l.self.q.rows(q, x, n)
|
||||
l.self.k.rows(k, x, n)
|
||||
@@ -285,12 +308,12 @@ func (m *Model) encode(ids []int) []float32 {
|
||||
}
|
||||
l.self.o.rows(tmp, ctxv, n)
|
||||
addNorm(x, tmp, l.selfNorm, n)
|
||||
hidden := make([]float32, n*l.fc1.out)
|
||||
l.fc1.rows(hidden, x, n)
|
||||
for i := range hidden {
|
||||
hidden[i] = m.activation(hidden[i])
|
||||
h := hidden[:n*l.fc1.out]
|
||||
l.fc1.rows(h, x, n)
|
||||
for i := range h {
|
||||
h[i] = m.activation(h[i])
|
||||
}
|
||||
l.fc2.rows(tmp, hidden, n)
|
||||
l.fc2.rows(tmp, h, n)
|
||||
addNorm(x, tmp, l.finalLayerNorm, n)
|
||||
}
|
||||
return x
|
||||
|
||||
+44
-24
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode/utf16"
|
||||
|
||||
@@ -34,8 +35,11 @@ const (
|
||||
var fontNames = []string{"Courier", "Courier-Bold", "Courier-Oblique", "Courier-BoldOblique"}
|
||||
|
||||
// Render writes pages to w. The title page, if any, must be the first page
|
||||
// and have Number 0 so that it is not numbered.
|
||||
func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
||||
// and have Number 0 so that it is not numbered. It returns, sorted, the
|
||||
// characters the standard Courier fonts cannot draw: each one is written
|
||||
// as "?" in the output, so the caller can warn instead of corrupting the
|
||||
// script silently.
|
||||
func Render(w io.Writer, pages []layout.Page, opt Options) ([]rune, error) {
|
||||
geo := layout.PaperGeometry(opt.Paper)
|
||||
var objs [][]byte // objs[i] is object number i+1
|
||||
add := func(s string) int {
|
||||
@@ -50,11 +54,12 @@ func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
||||
}
|
||||
add(fmt.Sprintf("<< /Title %s /Author %s /Creator (Plotzilla) /Producer (Plotzilla) >>", textString(opt.Title), textString(opt.Author)))
|
||||
|
||||
missing := map[rune]bool{}
|
||||
var kids []string
|
||||
for _, pg := range pages {
|
||||
stream, err := deflate(pageContent(pg, geo, opt.Revisions))
|
||||
stream, err := deflate(pageContent(pg, geo, opt.Revisions, missing))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
content := add(fmt.Sprintf("<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream", len(stream), stream))
|
||||
page := add(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %g %g] "+
|
||||
@@ -80,7 +85,16 @@ func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
||||
}
|
||||
fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R /Info 7 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objs)+1, xref)
|
||||
_, err := w.Write(buf.Bytes())
|
||||
return err
|
||||
return sortedRunes(missing), err
|
||||
}
|
||||
|
||||
func sortedRunes(set map[rune]bool) []rune {
|
||||
out := make([]rune, 0, len(set))
|
||||
for r := range set {
|
||||
out = append(out, r)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// points converts inches to PDF points, rounded to 1/100 pt.
|
||||
@@ -92,16 +106,16 @@ func baseline(geo layout.Geometry, row int) float64 {
|
||||
return top - float64(row)*fontSize - 9
|
||||
}
|
||||
|
||||
func text(b *bytes.Buffer, x, y float64, s string) {
|
||||
fmt.Fprintf(b, "BT /F1 12 Tf %.2f %.2f Td %s Tj ET\n", x, y, pdfString(s))
|
||||
func text(b *bytes.Buffer, x, y float64, s string, missing map[rune]bool) {
|
||||
fmt.Fprintf(b, "BT /F1 12 Tf %.2f %.2f Td %s Tj ET\n", x, y, pdfString(s, missing))
|
||||
}
|
||||
|
||||
func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Revision) []byte {
|
||||
func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Revision, missing map[rune]bool) []byte {
|
||||
var b bytes.Buffer
|
||||
header := (geo.Height-0.5)*ptPerIn - 9
|
||||
if pg.Number > 0 && pg.Label != "" && pg.Label != "1" {
|
||||
num := pg.Label + "."
|
||||
text(&b, layout.RightEdge*ptPerIn-float64(len([]rune(num)))*fontSize*0.6, header, num)
|
||||
text(&b, layout.RightEdge*ptPerIn-float64(len([]rune(num)))*fontSize*0.6, header, num, missing)
|
||||
}
|
||||
// A revised page names its revision; its latest changes are starred.
|
||||
rev := pg.Rev()
|
||||
@@ -114,17 +128,17 @@ func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Rev
|
||||
if r.Date != "" {
|
||||
label += " - " + r.Date
|
||||
}
|
||||
text(&b, 1.5*ptPerIn, header, label)
|
||||
text(&b, 1.5*ptPerIn, header, label, missing)
|
||||
}
|
||||
for _, ln := range pg.Lines {
|
||||
y := baseline(geo, ln.Row)
|
||||
x := ln.X * ptPerIn
|
||||
if ln.SceneNumber != "" {
|
||||
text(&b, 0.75*ptPerIn, y, ln.SceneNumber)
|
||||
text(&b, geo.SceneNumber*ptPerIn, y, ln.SceneNumber)
|
||||
text(&b, 0.75*ptPerIn, y, ln.SceneNumber, missing)
|
||||
text(&b, geo.SceneNumber*ptPerIn, y, ln.SceneNumber, missing)
|
||||
}
|
||||
if rev > 0 && ln.Rev == rev {
|
||||
text(&b, geo.RevisionMark*ptPerIn, y, "*")
|
||||
text(&b, geo.RevisionMark*ptPerIn, y, "*", missing)
|
||||
}
|
||||
fmt.Fprintf(&b, "BT %.2f %.2f Td ", x, y)
|
||||
cursor := x
|
||||
@@ -142,7 +156,7 @@ func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Rev
|
||||
underlines = append(underlines, [2]float64{cursor, cursor + width})
|
||||
}
|
||||
cursor += width
|
||||
fmt.Fprintf(&b, "/F%d 12 Tf %s Tj ", font, pdfString(s.Text))
|
||||
fmt.Fprintf(&b, "/F%d 12 Tf %s Tj ", font, pdfString(s.Text, missing))
|
||||
}
|
||||
b.WriteString("ET\n")
|
||||
for _, u := range underlines {
|
||||
@@ -164,12 +178,16 @@ func deflate(data []byte) ([]byte, error) {
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
// pdfString encodes s as a WinAnsi literal string for the Courier fonts.
|
||||
func pdfString(s string) string {
|
||||
// pdfString encodes s as a WinAnsi literal string for the Courier fonts,
|
||||
// recording in missing the characters they cannot draw.
|
||||
func pdfString(s string, missing map[rune]bool) string {
|
||||
var b strings.Builder
|
||||
b.WriteByte('(')
|
||||
for _, r := range s {
|
||||
c := winAnsi(r)
|
||||
c, ok := winAnsi(r)
|
||||
if !ok {
|
||||
missing[r] = true
|
||||
}
|
||||
if c == '(' || c == ')' || c == '\\' {
|
||||
b.WriteByte('\\')
|
||||
}
|
||||
@@ -189,7 +207,7 @@ func textString(s string) string {
|
||||
}
|
||||
}
|
||||
if ascii {
|
||||
return pdfString(s)
|
||||
return pdfString(s, map[rune]bool{}) // ASCII only: nothing to record
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("<FEFF")
|
||||
@@ -207,18 +225,20 @@ var cp1252 = map[rune]byte{
|
||||
'˜': 0x98, '™': 0x99, 'š': 0x9a, '›': 0x9b, 'œ': 0x9c, 'ž': 0x9e, 'Ÿ': 0x9f,
|
||||
}
|
||||
|
||||
func winAnsi(r rune) byte {
|
||||
// winAnsi encodes r as a WinAnsi byte; ok is false when the standard
|
||||
// fonts cannot draw r and '?' is written instead.
|
||||
func winAnsi(r rune) (c byte, ok bool) {
|
||||
switch {
|
||||
case r >= 0x20 && r < 0x7f:
|
||||
return byte(r)
|
||||
return byte(r), true
|
||||
case r >= 0xa0 && r <= 0xff:
|
||||
return byte(r)
|
||||
return byte(r), true
|
||||
}
|
||||
if c, ok := cp1252[r]; ok {
|
||||
return c
|
||||
return c, true
|
||||
}
|
||||
if r == ' ' || r == ' ' {
|
||||
return ' ' // narrow spaces used before French punctuation
|
||||
return ' ', true // narrow spaces used before French punctuation
|
||||
}
|
||||
return '?'
|
||||
return '?', false
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"compress/zlib"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -39,9 +40,13 @@ func TestRender(t *testing.T) {
|
||||
pages := []layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)}
|
||||
pages = append(pages, layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
||||
var buf bytes.Buffer
|
||||
if err := Render(&buf, pages, Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
||||
replaced, err := Render(&buf, pages, Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(replaced) != 0 {
|
||||
t.Errorf("the French sample is fully representable, replaced = %q", string(replaced))
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.HasPrefix(out, "%PDF-1.4") || !strings.HasSuffix(out, "%%EOF\n") || !strings.Contains(out, "/Count 2") {
|
||||
t.Fatalf("malformed PDF:\n%.300s", out)
|
||||
@@ -61,7 +66,7 @@ func TestRevisionsAndA4(t *testing.T) {
|
||||
doc.Elements[1].Runs[0].Rev = 1
|
||||
var buf bytes.Buffer
|
||||
pages := layout.Paginate(doc, layout.Options{}).Pages
|
||||
if err := Render(&buf, pages, Options{Paper: doc.Settings.Paper, Revisions: doc.Settings.Revisions}); err != nil {
|
||||
if _, err := Render(&buf, pages, Options{Paper: doc.Settings.Paper, Revisions: doc.Settings.Revisions}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := uncompressed(t, buf.Bytes())
|
||||
@@ -74,7 +79,7 @@ func TestRevisionsAndA4(t *testing.T) {
|
||||
|
||||
// Without revisions in the options, nothing is marked.
|
||||
buf.Reset()
|
||||
if err := Render(&buf, pages, Options{Paper: doc.Settings.Paper}); err != nil {
|
||||
if _, err := Render(&buf, pages, Options{Paper: doc.Settings.Paper}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stream := uncompressed(t, buf.Bytes()); strings.Contains(stream, "Revision") || strings.Contains(stream, "(*)") {
|
||||
@@ -82,6 +87,34 @@ func TestRevisionsAndA4(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplacedChars: characters outside WinAnsi are written as '?' and
|
||||
// reported, so an export can warn instead of corrupting the text silently.
|
||||
func TestReplacedChars(t *testing.T) {
|
||||
src := "INT. ΚΑΦΕΝΕΙΟ - ΜΕΡΑ\n\nΚαλημέρα… Plain ASCII.\n"
|
||||
doc := fountain.Parse(src)
|
||||
pages := layout.Paginate(doc, layout.Options{}).Pages
|
||||
var buf bytes.Buffer
|
||||
replaced, err := Render(&buf, pages, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := string(replaced)
|
||||
for _, r := range src { // every Greek letter is reported
|
||||
if r > 0x7f && r != '…' && !strings.ContainsRune(got, r) {
|
||||
t.Errorf("%q not reported in %q", r, got)
|
||||
}
|
||||
}
|
||||
if strings.ContainsRune(got, '…') { // WinAnsi has "…"
|
||||
t.Errorf("representable … reported in %q", got)
|
||||
}
|
||||
if !slices.IsSorted(replaced) {
|
||||
t.Errorf("replaced not sorted: %q", got)
|
||||
}
|
||||
if stream := uncompressed(t, buf.Bytes()); !strings.Contains(stream, "????????") {
|
||||
t.Errorf("Greek not written as '?':\n%s", stream)
|
||||
}
|
||||
}
|
||||
|
||||
// uncompressed returns the content streams of a PDF written by Render.
|
||||
func uncompressed(t *testing.T, pdf []byte) string {
|
||||
var out strings.Builder
|
||||
|
||||
@@ -90,6 +90,12 @@ func (c *Checker) Add(word string) {
|
||||
|
||||
func normalize(w string) string { return strings.ReplaceAll(w, "’", "'") }
|
||||
|
||||
// cacheLimit bounds the remembered answers, which are keyed on user input:
|
||||
// a long session of typos would grow the map forever otherwise. When full,
|
||||
// the cache is cleared and refilled; a script repeats the same words, so
|
||||
// re-checks are cheap.
|
||||
const cacheLimit = 10000
|
||||
|
||||
// Check reports whether word is correct. Words with digits ("3rd", "R2D2")
|
||||
// and single letters are always accepted.
|
||||
func (c *Checker) Check(word string) bool {
|
||||
@@ -104,6 +110,9 @@ func (c *Checker) Check(word string) bool {
|
||||
}
|
||||
ok, found := c.cache[word]
|
||||
if !found {
|
||||
if len(c.cache) >= cacheLimit {
|
||||
clear(c.cache)
|
||||
}
|
||||
ok = c.dict.Check(word)
|
||||
c.cache[word] = ok
|
||||
}
|
||||
|
||||
@@ -180,6 +180,35 @@ func TestLatin1(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckerCacheBound(t *testing.T) {
|
||||
d, err := Parse([]byte("SET UTF-8\n"), []byte("1\nknown\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := NewChecker(d, nil)
|
||||
// letters encodes n in bijective base 26: distinct pure-letter words.
|
||||
letters := func(n int) string {
|
||||
var w []rune
|
||||
for {
|
||||
w = append([]rune{rune('a' + n%26)}, w...)
|
||||
if n < 26 {
|
||||
return string(w)
|
||||
}
|
||||
n = n/26 - 1
|
||||
}
|
||||
}
|
||||
for i := range 2 * cacheLimit {
|
||||
c.Check(letters(i))
|
||||
}
|
||||
if len(c.cache) > cacheLimit {
|
||||
t.Errorf("cache holds %d words, limit %d", len(c.cache), cacheLimit)
|
||||
}
|
||||
// The answers are unchanged by a clear.
|
||||
if !c.Check("known") || c.Check("knownn") {
|
||||
t.Error("wrong answers after the cache cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWords(t *testing.T) {
|
||||
got := Words("Élise's well-known R2D2, l’homme… 'quoted' x")
|
||||
want := []Word{
|
||||
|
||||
@@ -29,9 +29,7 @@ func main() {
|
||||
app := NewApp()
|
||||
for _, arg := range os.Args[1:] {
|
||||
if !strings.HasPrefix(arg, "-") {
|
||||
if _, err := os.Stat(arg); err == nil {
|
||||
app.initialPath = arg
|
||||
}
|
||||
app.initialPath = arg // may not exist yet: then it is a new script
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -49,7 +47,7 @@ func main() {
|
||||
BackgroundColour: &options.RGBA{R: 0, G: 0, B: 0, A: 0},
|
||||
OnStartup: app.startup,
|
||||
OnBeforeClose: app.beforeClose,
|
||||
Bind: []interface{}{app},
|
||||
Bind: []any{app},
|
||||
Linux: &linux.Options{
|
||||
ProgramName: "plotzilla",
|
||||
Icon: icon,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Maintainer: Sandro Cazzaniga <sandro@cazzaniga.fr>
|
||||
|
||||
pkgname=plotzilla
|
||||
pkgver=1.2
|
||||
pkgver=1.4.1
|
||||
pkgrel=1
|
||||
pkgdesc='Screenwriting app for Linux: scripts in the Hollywood standard format, Fountain, FDX and PDF'
|
||||
arch=('x86_64' 'aarch64')
|
||||
|
||||
@@ -277,7 +277,7 @@ func pdfOutput(src, tmp, path string) error {
|
||||
pages := append([]layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)},
|
||||
layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
||||
var b bytes.Buffer
|
||||
if err := pdf.Render(&b, pages, pdf.Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
||||
if _, err := pdf.Render(&b, pages, pdf.Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
||||
return err
|
||||
}
|
||||
file := filepath.Join(tmp, "sample.pdf")
|
||||
|
||||
+33
-21
@@ -35,7 +35,7 @@ const (
|
||||
)
|
||||
|
||||
// errStalled aborts a download that receives nothing for stallTimeout.
|
||||
var errStalled = errors.New("the download stalled (no data received for 30 seconds)")
|
||||
var errStalled = fmt.Errorf("the download stalled (no data received for %s)", stallTimeout)
|
||||
|
||||
var modelFiles = []marian.RemoteFile{
|
||||
{Name: "config.json", Size: 1416, SHA256: "b3be13d046d9899d7aab8cf4ed624d9a79f5776038ba793f6b4d2ce3e02192f7"},
|
||||
@@ -232,10 +232,8 @@ func (a *App) Translate(doc screenplay.Document, elements []int, titlePage bool)
|
||||
res, err := translate.Document(ctx, &doc, model, lexicon, translate.Options{
|
||||
Elements: elements,
|
||||
TitlePage: titlePage,
|
||||
// The model splits its largest product across every CPU already:
|
||||
// half as many sentences at once keeps the machine responsive.
|
||||
Workers: max(1, runtime.NumCPU()/2),
|
||||
Progress: func(done, total int) { emit(int64(done), int64(total)) },
|
||||
Workers: translateWorkers(),
|
||||
Progress: func(done, total int) { emit(int64(done), int64(total)) },
|
||||
})
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil, nil
|
||||
@@ -246,6 +244,10 @@ func (a *App) Translate(doc screenplay.Document, elements []int, titlePage bool)
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// translateWorkers is how many sentences are translated at once: half the
|
||||
// CPUs, and each one splits its decoding steps across its share of them.
|
||||
func translateWorkers() int { return max(1, runtime.NumCPU()/2) }
|
||||
|
||||
// translator returns the model and the dictionaries that help tell French
|
||||
// from English, loading them the first time. The load happens without
|
||||
// a.tr.mu: reading 300 MB takes seconds, and CancelTranslation must stay
|
||||
@@ -271,7 +273,10 @@ func (a *App) translator(ctx context.Context) (*marian.Model, translate.Lexicon,
|
||||
}
|
||||
return nil, nil, fmt.Errorf("loading the translation model: %w", err)
|
||||
}
|
||||
lexicon = loadLexicon()
|
||||
// Every sentence worker gets an equal share of the CPUs for its
|
||||
// decoding steps, so workers × threads never oversubscribe the machine.
|
||||
m.SetThreads(max(1, runtime.NumCPU()/translateWorkers()))
|
||||
lexicon = a.loadLexicon()
|
||||
a.tr.mu.Lock()
|
||||
if ctx.Err() == nil { // cancelled while loading: cache nothing
|
||||
a.tr.model, a.tr.lexicon = m, lexicon
|
||||
@@ -311,25 +316,32 @@ func (l lexicon) Knows(word string) (french, english bool) { return l.fr.Check(w
|
||||
|
||||
// loadLexicon needs both a French and an English dictionary: with one
|
||||
// only, every word it knows would count for its language. Without them,
|
||||
// detection relies on function words and accents.
|
||||
func loadLexicon() translate.Lexicon {
|
||||
// detection relies on function words and accents. The dictionaries come
|
||||
// from the same cache as the spell checker's: a Dictionary is read-only
|
||||
// once loaded, so it can be shared.
|
||||
func (a *App) loadLexicon() translate.Lexicon {
|
||||
langs := spell.Available(dictionaryDirs())
|
||||
load := func(lang, preferred string) *spell.Dictionary {
|
||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred })
|
||||
if i < 0 {
|
||||
i = slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") })
|
||||
pick := func(lang, preferred string) string {
|
||||
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred }); i >= 0 {
|
||||
return langs[i].Code
|
||||
}
|
||||
if i < 0 {
|
||||
return nil
|
||||
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") }); i >= 0 {
|
||||
return langs[i].Code
|
||||
}
|
||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return d
|
||||
return ""
|
||||
}
|
||||
fr, en := load("fr", "fr_FR"), load("en", "en_US")
|
||||
if fr == nil || en == nil {
|
||||
frCode, enCode := pick("fr", "fr_FR"), pick("en", "en_US")
|
||||
if frCode == "" || enCode == "" {
|
||||
return nil
|
||||
}
|
||||
a.spellMu.Lock()
|
||||
defer a.spellMu.Unlock()
|
||||
fr, err := a.dictionaryLocked(frCode)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
en, err := a.dictionaryLocked(enCode)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return lexicon{fr, en}
|
||||
|
||||
Reference in new issue
Block a user