Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d355d93126 | ||
|
|
2de0d59641 | ||
|
|
093f7ea10d | ||
|
|
764c9d4dc3 | ||
|
|
6622203506 | ||
|
|
b0bcb812b1 | ||
|
|
823c9f2d0f | ||
|
|
343836df6b | ||
|
|
e85d0ba7e9 | ||
|
|
1440adc199 | ||
|
|
82bfef8cba | ||
|
|
d6ee57130f | ||
|
|
e80ad634ef | ||
|
|
5e7ae97b57 | ||
|
|
c0a52a9040 | ||
|
|
643adffb12 | ||
|
|
b6b58c7aca | ||
|
|
839e9291d6 | ||
|
|
8181c6e885 | ||
|
|
082152fc31 | ||
|
|
b84284b60c | ||
|
|
7872403e11 |
No files matched your search
@@ -46,8 +46,9 @@ behaviour or shortcuts changed.
|
|||||||
```
|
```
|
||||||
main.go Wails bootstrap: frameless translucent window, LANGUAGE=en, CLI file argument
|
main.go Wails bootstrap: frameless translucent window, LANGUAGE=en, CLI file argument
|
||||||
app.go App struct bound to JS: file I/O, dialogs, pagination, autosave recovery
|
app.go App struct bound to JS: file I/O, dialogs, pagination, autosave recovery
|
||||||
translation.go App methods for translation: model status/download (pinned files + SHA-256),
|
translation.go App methods for translation: model status/download (pinned files + SHA-256,
|
||||||
Translate, CancelTranslation, model loaded lazily and freed after 5 min idle
|
resumable), Translate, CancelTranslation, model loaded lazily and freed
|
||||||
|
after 5 min idle
|
||||||
desktop_linux.go cgo: desktop light/dark preference for GTK dialogs, window button layout, compositing
|
desktop_linux.go cgo: desktop light/dark preference for GTK dialogs, window button layout, compositing
|
||||||
desktop/ app icon (plotzilla.svg, source of truth; plotzilla.png = 256 px render embedded as
|
desktop/ app icon (plotzilla.svg, source of truth; plotzilla.png = 256 px render embedded as
|
||||||
the X11 window icon), plotzilla.desktop, MIME types for .fountain/.fdx
|
the X11 window icon), plotzilla.desktop, MIME types for .fountain/.fdx
|
||||||
@@ -61,7 +62,8 @@ internal/layout line wrapping + pagination (paper sizes, locked pages);
|
|||||||
internal/pdf hand-written PDF 1.4 writer, standard Courier fonts, zero dependencies
|
internal/pdf hand-written PDF 1.4 writer, standard Courier fonts, zero dependencies
|
||||||
internal/spell Hunspell .aff/.dic reader, checker (compounds too), suggestions, word scan
|
internal/spell Hunspell .aff/.dic reader, checker (compounds too), suggestions, word scan
|
||||||
internal/marian MarianMT (OPUS-MT) inference in pure Go: safetensors, SentencePiece unigram,
|
internal/marian MarianMT (OPUS-MT) inference in pure Go: safetensors, SentencePiece unigram,
|
||||||
transformer encoder-decoder, beam search, verified download
|
transformer encoder-decoder, beam search, verified download. In-tree on
|
||||||
|
purpose (one binary, no cgo, offline by design): see the package doc.
|
||||||
internal/translate French → English script translation: language detection, screenplay term
|
internal/translate French → English script translation: language detection, screenplay term
|
||||||
tables, per-element rules, name protection, parallel sentence translation
|
tables, per-element rules, name protection, parallel sentence translation
|
||||||
frontend/index.html app shell: header bar, navigator, paper, HUD, <dialog>s
|
frontend/index.html app shell: header bar, navigator, paper, HUD, <dialog>s
|
||||||
@@ -80,7 +82,7 @@ frontend/src/icons.ts inline Lucide-style SVG icons
|
|||||||
frontend/src/style.css the whole UI theme (dark, blue) and the on-screen page geometry
|
frontend/src/style.css the whole UI theme (dark, blue) and the on-screen page geometry
|
||||||
screenshots/ README images (2880×1800 UI captures + pdf-output.png)
|
screenshots/ README images (2880×1800 UI captures + pdf-output.png)
|
||||||
tools/screenshots/ `make screenshots`: sample script, mock server (main.go), Firefox capture (capture.mjs)
|
tools/screenshots/ `make screenshots`: sample script, mock server (main.go), Firefox capture (capture.mjs)
|
||||||
VERSION current version (1.0)
|
VERSION current version
|
||||||
CHANGELOG.md Keep a Changelog; sections Added, Changed, Fixed only
|
CHANGELOG.md Keep a Changelog; sections Added, Changed, Fixed only
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -211,11 +213,16 @@ change `style.css` too** (and vice versa).
|
|||||||
- Translation (`Ctrl+Shift+L`) runs in Go, offline: `internal/marian` runs the OPUS-MT fr-en
|
- Translation (`Ctrl+Shift+L`) runs in Go, offline: `internal/marian` runs the OPUS-MT fr-en
|
||||||
model (Hugging Face form: `config.json`, `model.safetensors`, `source.spm`, `target.spm`,
|
model (Hugging Face form: `config.json`, `model.safetensors`, `source.spm`, `target.spm`,
|
||||||
`vocab.json`), downloaded on demand into `~/.local/share/plotzilla/models/opus-mt-fr-en`
|
`vocab.json`), downloaded on demand into `~/.local/share/plotzilla/models/opus-mt-fr-en`
|
||||||
(also looked up in `$XDG_DATA_DIRS/plotzilla/models/`). `translate.Document` returns the
|
(also looked up in `$XDG_DATA_DIRS/plotzilla/models/`). The download resumes the `.partial`
|
||||||
|
files an interruption left, skips files already downloaded, and aborts when nothing arrives
|
||||||
|
for 30 s; a directory only counts as installed when every file has its pinned size.
|
||||||
|
`translate.Document` returns the
|
||||||
changed elements (flat indexes) and the title page; the frontend replaces them in one
|
changed elements (flat indexes) and the title page; the frontend replaces them in one
|
||||||
transaction (`applyTranslation`), only if the doc is unchanged. Both tasks report progress
|
transaction (`applyTranslation`), only if the doc is unchanged. Both tasks report progress
|
||||||
with the `translation-progress` event (done, total; `0, 0` while loading) and stop on
|
with the `translation-progress` event (done, total; `0, 0` while loading) and stop on
|
||||||
`CancelTranslation`, returning `nil`/`false` (not an error) when cancelled.
|
`CancelTranslation`, returning `nil`/`false` (not an error) when cancelled. The model loads
|
||||||
|
without holding the translation lock (and `marian.Load` takes the context) so Cancel stays
|
||||||
|
responsive during the seconds-long load; `CancelTranslation` waits for the task to wind down.
|
||||||
- Find & replace is frontend-only (`search.ts`): matches are computed on `blockText`, where one
|
- Find & replace is frontend-only (`search.ts`): matches are computed on `blockText`, where one
|
||||||
character is one document position. Replacements go through `replaceText()` (commands.ts),
|
character is one document position. Replacements go through `replaceText()` (commands.ts),
|
||||||
which keeps the marks, and each one is its own undo step (`closeHistory`).
|
which keeps the marks, and each one is its own undo step (`closeHistory`).
|
||||||
|
|||||||
@@ -5,6 +5,43 @@ All notable changes to Plotzilla are documented in this file.
|
|||||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The current
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The current
|
||||||
version is in the `VERSION` file.
|
version is in the `VERSION` file.
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [1.4.1] - 2026-10-15
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- The PDF export warns when the script contains characters the standard PDF fonts
|
||||||
|
cannot draw: they are written as `?` in the file, and are now listed in a dialog
|
||||||
|
instead of being silently lost.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- `plotzilla new-script.fountain` opens a new script that will be saved to that file,
|
||||||
|
instead of an untitled document that forgot the name it was given.
|
||||||
|
- A Final Draft file named `.txt` (or anything but `.fdx`) is recognised by its
|
||||||
|
content and opened as FDX, not parsed as Fountain.
|
||||||
|
- The unsaved-changes marker is now updated atomically: an edit landing while a save
|
||||||
|
finished could make the close confirmation unreliable.
|
||||||
|
- Saving is now durable: the file is synced to disk before it replaces the old one (a
|
||||||
|
power cut just after a save could leave an empty script). An existing file keeps its
|
||||||
|
permissions, and saving through a symlink replaces the file it points to instead of
|
||||||
|
replacing the link.
|
||||||
|
- Translation is faster and uses much less memory: the beam search no longer copies
|
||||||
|
every beam's decoder state for every candidate at every step, and reuses its
|
||||||
|
work buffers instead of allocating new ones per layer and step. Each sentence
|
||||||
|
now uses its share of the processors, so translating a script no longer starts
|
||||||
|
more workers than the machine has.
|
||||||
|
|
||||||
|
## [1.4] - 2026-10-15
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The Cancel button responds while the translation model is loading.
|
||||||
|
- The translation model download resumes where it stopped after an interruption, skips files
|
||||||
|
already downloaded, and reports a stalled connection instead of hanging. A model directory
|
||||||
|
with missing or truncated files is offered for download again instead of failing to load.
|
||||||
|
|
||||||
## [1.3] - 2026-10-01
|
## [1.3] - 2026-10-01
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
@@ -318,9 +318,9 @@ The model is [OPUS-MT French–English](https://huggingface.co/Helsinki-NLP/opus
|
|||||||
University of Helsinki (Apache 2.0 license). Plotzilla runs it itself, in Go, with nothing else
|
University of Helsinki (Apache 2.0 license). Plotzilla runs it itself, in Go, with nothing else
|
||||||
to install. The first translation asks to download it (about 300 MB) from Hugging Face into
|
to install. The first translation asks to download it (about 300 MB) from Hugging Face into
|
||||||
`~/.local/share/plotzilla/models/opus-mt-fr-en`; every file is checked against its known
|
`~/.local/share/plotzilla/models/opus-mt-fr-en`; every file is checked against its known
|
||||||
checksum. The model takes about 300 MB of memory while it is loaded, and is unloaded after five
|
checksum, and an interrupted download resumes where it stopped. The model takes about 300 MB of
|
||||||
minutes without translating. A machine translation is a first draft: read it over, especially
|
memory while it is loaded, and is unloaded after five minutes without translating. A machine
|
||||||
the dialogue.
|
translation is a first draft: read it over, especially the dialogue.
|
||||||
|
|
||||||
## Pages and scenes
|
## Pages and scenes
|
||||||
|
|
||||||
@@ -473,7 +473,8 @@ The title page and first pages of a sample script, as exported:
|
|||||||

|

|
||||||
|
|
||||||
The standard Courier fonts cover Western European characters (accents, curly quotes, dashes).
|
The standard Courier fonts cover Western European characters (accents, curly quotes, dashes).
|
||||||
Other characters, such as Greek, Cyrillic or Asian scripts, are printed as `?`.
|
Other characters, such as Greek, Cyrillic or Asian scripts, are printed as `?`: the export then
|
||||||
|
warns and lists them, so they are not lost silently.
|
||||||
|
|
||||||
**Final Draft** (`Ctrl+Shift+E`): writes an `.fdx` file that opens in Final Draft and other apps
|
**Final Draft** (`Ctrl+Shift+E`): writes an `.fdx` file that opens in Final Draft and other apps
|
||||||
that read FDX.
|
that read FDX.
|
||||||
@@ -568,3 +569,6 @@ scripts in the PDF.
|
|||||||
|
|
||||||
Plotzilla is free software, released under the GNU General Public License v3.0. See
|
Plotzilla is free software, released under the GNU General Public License v3.0. See
|
||||||
[LICENSE](LICENSE).
|
[LICENSE](LICENSE).
|
||||||
|
|
||||||
|
The model downloaded for translation is separate: [OPUS-MT French–English](https://huggingface.co/Helsinki-NLP/opus-mt-fr-en),
|
||||||
|
by the University of Helsinki, is released under the Apache 2.0 license.
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||||
@@ -24,15 +25,20 @@ import (
|
|||||||
|
|
||||||
// App is bound to the frontend: every exported method is callable from JS.
|
// App is bound to the frontend: every exported method is callable from JS.
|
||||||
type App struct {
|
type App struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
dirty bool
|
// dirty decides whether closing the window asks for confirmation. It is
|
||||||
|
// written from Wails call goroutines (SetDirty) and read by the GTK
|
||||||
|
// close handler: atomic, like spellMu protects the checker.
|
||||||
|
dirty atomic.Bool
|
||||||
initialPath string
|
initialPath string
|
||||||
|
|
||||||
// Wails runs each call on its own goroutine: spelling calls share one
|
// Wails runs each call on its own goroutine: spelling calls share one
|
||||||
// loaded dictionary.
|
// loaded dictionary. dicts caches them per language so the translation's
|
||||||
|
// language detection does not read the same files again.
|
||||||
spellMu sync.Mutex
|
spellMu sync.Mutex
|
||||||
checker *spell.Checker
|
checker *spell.Checker
|
||||||
checkerLang string
|
checkerLang string
|
||||||
|
dicts map[string]*spell.Dictionary
|
||||||
|
|
||||||
tr translation
|
tr translation
|
||||||
}
|
}
|
||||||
@@ -74,7 +80,7 @@ func (a *App) startup(ctx context.Context) {
|
|||||||
// beforeClose keeps the window open while there are unsaved changes and
|
// beforeClose keeps the window open while there are unsaved changes and
|
||||||
// lets the frontend ask; it quits again once the user has decided.
|
// lets the frontend ask; it quits again once the user has decided.
|
||||||
func (a *App) beforeClose(ctx context.Context) bool {
|
func (a *App) beforeClose(ctx context.Context) bool {
|
||||||
if !a.dirty {
|
if !a.dirty.Load() {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
runtime.EventsEmit(a.ctx, "confirm-close")
|
runtime.EventsEmit(a.ctx, "confirm-close")
|
||||||
@@ -83,13 +89,18 @@ func (a *App) beforeClose(ctx context.Context) bool {
|
|||||||
|
|
||||||
func (a *App) NewDocument() *screenplay.Document { return screenplay.New() }
|
func (a *App) NewDocument() *screenplay.Document { return screenplay.New() }
|
||||||
|
|
||||||
// InitialFile returns the script passed on the command line, if any.
|
// InitialFile returns the script passed on the command line, if any. A
|
||||||
|
// path that does not exist yet opens a new script, written there on the
|
||||||
|
// first save: `plotzilla new-script.fountain` creates the file.
|
||||||
func (a *App) InitialFile() (*OpenResult, error) {
|
func (a *App) InitialFile() (*OpenResult, error) {
|
||||||
if a.initialPath == "" {
|
if a.initialPath == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
path := a.initialPath
|
path := a.initialPath
|
||||||
a.initialPath = ""
|
a.initialPath = ""
|
||||||
|
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||||
|
return &OpenResult{Path: path, Document: screenplay.New()}, nil
|
||||||
|
}
|
||||||
return a.load(path)
|
return a.load(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,7 +118,7 @@ func (a *App) load(path string) (*OpenResult, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var doc *screenplay.Document
|
var doc *screenplay.Document
|
||||||
if strings.EqualFold(filepath.Ext(path), ".fdx") {
|
if isFDX(path, data) {
|
||||||
if doc, err = fdx.Read(bytes.NewReader(data)); err != nil {
|
if doc, err = fdx.Read(bytes.NewReader(data)); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -117,6 +128,16 @@ func (a *App) load(path string) (*OpenResult, error) {
|
|||||||
return &OpenResult{Path: path, Document: doc}, nil
|
return &OpenResult{Path: path, Document: doc}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isFDX reports whether a file is a Final Draft one: the extension says
|
||||||
|
// so, or its content does — a ".txt" export of Final Draft is still FDX.
|
||||||
|
func isFDX(path string, data []byte) bool {
|
||||||
|
if strings.EqualFold(filepath.Ext(path), ".fdx") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
head := strings.TrimPrefix(string(data[:min(len(data), 512)]), "\ufeff")
|
||||||
|
return strings.Contains(head, "<FinalDraft")
|
||||||
|
}
|
||||||
|
|
||||||
// Save writes the document to path, asking for one when empty. It returns
|
// Save writes the document to path, asking for one when empty. It returns
|
||||||
// the path written, or "" if the user cancelled.
|
// the path written, or "" if the user cancelled.
|
||||||
func (a *App) Save(doc screenplay.Document, path string) (string, error) {
|
func (a *App) Save(doc screenplay.Document, path string) (string, error) {
|
||||||
@@ -147,11 +168,19 @@ func (a *App) SaveAs(doc screenplay.Document, current string) (string, error) {
|
|||||||
return a.Save(doc, path)
|
return a.Save(doc, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current string) (string, error) {
|
// ExportResult describes a written PDF export.
|
||||||
|
type ExportResult struct {
|
||||||
|
Path string `json:"path"` // "" when the user cancelled the dialog
|
||||||
|
// Replaced are the characters the standard PDF fonts cannot draw:
|
||||||
|
// each one is written as "?" in the file.
|
||||||
|
Replaced []string `json:"replaced,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current string) (ExportResult, error) {
|
||||||
path, err := a.saveDialog("Export PDF", current, ".pdf",
|
path, err := a.saveDialog("Export PDF", current, ".pdf",
|
||||||
[]runtime.FileFilter{{DisplayName: "PDF (*.pdf)", Pattern: "*.pdf"}})
|
[]runtime.FileFilter{{DisplayName: "PDF (*.pdf)", Pattern: "*.pdf"}})
|
||||||
if err != nil || path == "" {
|
if err != nil || path == "" {
|
||||||
return "", err
|
return ExportResult{}, err
|
||||||
}
|
}
|
||||||
var pages []layout.Page
|
var pages []layout.Page
|
||||||
if opts.TitlePage && !doc.TitlePage.IsEmpty() {
|
if opts.TitlePage && !doc.TitlePage.IsEmpty() {
|
||||||
@@ -163,11 +192,18 @@ func (a *App) ExportPDF(doc screenplay.Document, opts ExportOptions, current str
|
|||||||
pdfOpts.Revisions = doc.Settings.Revisions
|
pdfOpts.Revisions = doc.Settings.Revisions
|
||||||
}
|
}
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
err = pdf.Render(&buf, pages, pdfOpts)
|
replaced, err := pdf.Render(&buf, pages, pdfOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return ExportResult{}, err
|
||||||
}
|
}
|
||||||
return path, writeAtomic(path, buf.Bytes())
|
if err := writeAtomic(path, buf.Bytes()); err != nil {
|
||||||
|
return ExportResult{}, err
|
||||||
|
}
|
||||||
|
res := ExportResult{Path: path}
|
||||||
|
for _, r := range replaced {
|
||||||
|
res.Replaced = append(res.Replaced, string(r))
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) ExportFDX(doc screenplay.Document, current string) (string, error) {
|
func (a *App) ExportFDX(doc screenplay.Document, current string) (string, error) {
|
||||||
@@ -201,7 +237,7 @@ func (a *App) RevisionSet(n int) screenplay.Revision {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) SetDirty(dirty bool) { a.dirty = dirty }
|
func (a *App) SetDirty(dirty bool) { a.dirty.Store(dirty) }
|
||||||
|
|
||||||
// dataDir is the XDG data directory (~/.local/share).
|
// dataDir is the XDG data directory (~/.local/share).
|
||||||
func dataDir() string {
|
func dataDir() string {
|
||||||
@@ -275,6 +311,28 @@ func (a *App) SpellLanguages() SpellInfo {
|
|||||||
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
return SpellInfo{Languages: codes, Default: spell.Preferred(codes, locale)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dictionaryLocked returns the Hunspell dictionary for a language code
|
||||||
|
// ("en_US"), loading it the first time. spellMu must be held.
|
||||||
|
func (a *App) dictionaryLocked(code string) (*spell.Dictionary, error) {
|
||||||
|
if d, ok := a.dicts[code]; ok {
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
langs := spell.Available(dictionaryDirs())
|
||||||
|
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == code })
|
||||||
|
if i < 0 {
|
||||||
|
return nil, fmt.Errorf("no %s dictionary installed", code)
|
||||||
|
}
|
||||||
|
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("loading the %s dictionary: %w", code, err)
|
||||||
|
}
|
||||||
|
if a.dicts == nil {
|
||||||
|
a.dicts = map[string]*spell.Dictionary{}
|
||||||
|
}
|
||||||
|
a.dicts[code] = d
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
// spellChecker returns the checker for lang, loading its dictionary the
|
// spellChecker returns the checker for lang, loading its dictionary the
|
||||||
// first time it is asked for.
|
// first time it is asked for.
|
||||||
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
||||||
@@ -283,15 +341,9 @@ func (a *App) spellChecker(lang string) (*spell.Checker, error) {
|
|||||||
if a.checker != nil && a.checkerLang == lang {
|
if a.checker != nil && a.checkerLang == lang {
|
||||||
return a.checker, nil
|
return a.checker, nil
|
||||||
}
|
}
|
||||||
langs := spell.Available(dictionaryDirs())
|
dict, err := a.dictionaryLocked(lang)
|
||||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == lang })
|
|
||||||
if i < 0 {
|
|
||||||
return nil, fmt.Errorf("no %s dictionary installed", lang)
|
|
||||||
}
|
|
||||||
l := langs[i]
|
|
||||||
dict, err := spell.Load(l.Aff, l.Dic)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("loading the %s dictionary: %w", lang, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
var personal []string
|
var personal []string
|
||||||
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
if data, err := os.ReadFile(personalDictionaryPath()); err == nil {
|
||||||
@@ -382,20 +434,48 @@ func (a *App) saveDialog(title, current, ext string, filters []runtime.FileFilte
|
|||||||
return path, nil
|
return path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeAtomic writes data to path atomically and durably: a temporary file
|
||||||
|
// in the same directory is written, given the file's permissions and synced
|
||||||
|
// to disk, then renamed over path and the directory itself synced, so a
|
||||||
|
// power cut just after a save cannot leave an empty script behind (the very
|
||||||
|
// failure autosave, which calls this every 1.2 s, exists to prevent). A
|
||||||
|
// symlinked path is resolved first: the file it points to is replaced, not
|
||||||
|
// the link. An existing file keeps its permissions; a new one gets 0o644.
|
||||||
func writeAtomic(path string, data []byte) error {
|
func writeAtomic(path string, data []byte) error {
|
||||||
|
if target, err := filepath.EvalSymlinks(path); err == nil {
|
||||||
|
path = target
|
||||||
|
}
|
||||||
|
mode := os.FileMode(0o644)
|
||||||
|
if st, err := os.Stat(path); err == nil {
|
||||||
|
mode = st.Mode().Perm()
|
||||||
|
}
|
||||||
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
|
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, werr := tmp.Write(data)
|
if _, err = tmp.Write(data); err == nil {
|
||||||
cerr := tmp.Close()
|
err = tmp.Chmod(mode)
|
||||||
if err := errors.Join(werr, cerr); err != nil {
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = tmp.Sync()
|
||||||
|
}
|
||||||
|
if cerr := tmp.Close(); err == nil {
|
||||||
|
err = cerr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
os.Remove(tmp.Name())
|
os.Remove(tmp.Name())
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||||
os.Remove(tmp.Name())
|
os.Remove(tmp.Name())
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return os.Rename(tmp.Name(), path)
|
// The rename is a directory entry: sync the directory so it too
|
||||||
|
// survives a crash.
|
||||||
|
dir, err := os.Open(filepath.Dir(path))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer dir.Close()
|
||||||
|
return dir.Sync()
|
||||||
}
|
}
|
||||||
Generated
+3
-3
@@ -1155,9 +1155,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/source-map-js": {
|
"node_modules/source-map-js": {
|
||||||
"version": "1.2.1",
|
"version": "1.2.2",
|
||||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz",
|
||||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
"integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "BSD-3-Clause",
|
"license": "BSD-3-Clause",
|
||||||
"engines": {
|
"engines": {
|
||||||
|
|||||||
@@ -144,13 +144,21 @@ export interface ExportOptions {
|
|||||||
revisions: boolean
|
revisions: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A written PDF export. */
|
||||||
|
export interface ExportResult {
|
||||||
|
/** Empty when the user cancelled the dialog. */
|
||||||
|
path: string
|
||||||
|
/** Characters the standard PDF fonts cannot draw, written as "?" in the file. */
|
||||||
|
replaced?: string[]
|
||||||
|
}
|
||||||
|
|
||||||
interface GoApp {
|
interface GoApp {
|
||||||
NewDocument(): Promise<ScriptDocument>
|
NewDocument(): Promise<ScriptDocument>
|
||||||
InitialFile(): Promise<OpenResult | null>
|
InitialFile(): Promise<OpenResult | null>
|
||||||
Open(): Promise<OpenResult | null>
|
Open(): Promise<OpenResult | null>
|
||||||
Save(doc: ScriptDocument, path: string): Promise<string>
|
Save(doc: ScriptDocument, path: string): Promise<string>
|
||||||
SaveAs(doc: ScriptDocument, current: string): Promise<string>
|
SaveAs(doc: ScriptDocument, current: string): Promise<string>
|
||||||
ExportPDF(doc: ScriptDocument, opts: ExportOptions, current: string): Promise<string>
|
ExportPDF(doc: ScriptDocument, opts: ExportOptions, current: string): Promise<ExportResult>
|
||||||
ExportFDX(doc: ScriptDocument, current: string): Promise<string>
|
ExportFDX(doc: ScriptDocument, current: string): Promise<string>
|
||||||
Paginate(doc: ScriptDocument): Promise<LayoutInfo>
|
Paginate(doc: ScriptDocument): Promise<LayoutInfo>
|
||||||
ParseFountain(text: string): Promise<Element[] | null>
|
ParseFountain(text: string): Promise<Element[] | null>
|
||||||
|
|||||||
@@ -947,7 +947,15 @@ const actions: Record<string, () => unknown> = {
|
|||||||
renderPaper()
|
renderPaper()
|
||||||
settingsChanged()
|
settingsChanged()
|
||||||
}
|
}
|
||||||
await api.ExportPDF(script(), chosen, app.path).catch(reportError)
|
const res = await api.ExportPDF(script(), chosen, app.path).catch(reportError)
|
||||||
|
if (res?.replaced?.length) {
|
||||||
|
const list = res.replaced.map((c) => `"${c}"`).join(' ')
|
||||||
|
await messageBox({
|
||||||
|
title: 'Characters replaced in the PDF',
|
||||||
|
text: `The standard PDF fonts cannot draw these characters: ${list}. Each one is written as "?" in the exported file.`,
|
||||||
|
ok: 'OK',
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
dialog.showModal()
|
dialog.showModal()
|
||||||
},
|
},
|
||||||
|
|||||||
+64
-27
@@ -13,7 +13,9 @@ type beam struct {
|
|||||||
tokens []int // starts with the decoder start token
|
tokens []int // starts with the decoder start token
|
||||||
score float64
|
score float64
|
||||||
// Keys and values of the decoder's self-attention, one slice per layer
|
// Keys and values of the decoder's self-attention, one slice per layer
|
||||||
// holding len(tokens) rows: each step only computes its own row.
|
// holding len(tokens) rows: each step only computes its own row. When a
|
||||||
|
// beam is the only survivor of its parent it takes the parent's slices
|
||||||
|
// over instead of copying them (the parent is dead from that step on).
|
||||||
keys, values [][]float32
|
keys, values [][]float32
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,6 +29,16 @@ type candidate struct {
|
|||||||
score float64
|
score float64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// decoderScratch holds the work buffers of a decoding step, shared by the
|
||||||
|
// layers, the beams and the steps of a search: allocating them afresh for
|
||||||
|
// every layer of every step costs more than the dot products they serve.
|
||||||
|
type decoderScratch struct {
|
||||||
|
q, k, v []float32
|
||||||
|
ctxv, tmp []float32
|
||||||
|
scores []float32
|
||||||
|
hidden []float32
|
||||||
|
}
|
||||||
|
|
||||||
// beamSearch generates the translation, as Hugging Face's generate() does
|
// beamSearch generates the translation, as Hugging Face's generate() does
|
||||||
// with the model's defaults: num_beams beams, length penalty 1, the
|
// with the model's defaults: num_beams beams, length penalty 1, the
|
||||||
// padding token never produced.
|
// padding token never produced.
|
||||||
@@ -35,18 +47,32 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
|||||||
numBeams := max(m.cfg.NumBeams, 1)
|
numBeams := max(m.cfg.NumBeams, 1)
|
||||||
crossK := make([][]float32, len(m.decoder))
|
crossK := make([][]float32, len(m.decoder))
|
||||||
crossV := make([][]float32, len(m.decoder))
|
crossV := make([][]float32, len(m.decoder))
|
||||||
|
ffn := 0
|
||||||
for i, l := range m.decoder {
|
for i, l := range m.decoder {
|
||||||
crossK[i] = make([]float32, srcLen*d)
|
crossK[i] = make([]float32, srcLen*d)
|
||||||
crossV[i] = make([]float32, srcLen*d)
|
crossV[i] = make([]float32, srcLen*d)
|
||||||
l.cross.k.rows(crossK[i], enc, srcLen)
|
l.cross.k.rows(crossK[i], enc, srcLen)
|
||||||
l.cross.v.rows(crossV[i], enc, srcLen)
|
l.cross.v.rows(crossV[i], enc, srcLen)
|
||||||
|
ffn = max(ffn, l.fc1.out)
|
||||||
|
}
|
||||||
|
maxLen := min(m.cfg.MaxPositions-1, 2*srcLen+10)
|
||||||
|
scratch := &decoderScratch{
|
||||||
|
q: make([]float32, d), k: make([]float32, d), v: make([]float32, d),
|
||||||
|
ctxv: make([]float32, d), tmp: make([]float32, d),
|
||||||
|
scores: make([]float32, max(maxLen+1, srcLen)),
|
||||||
|
hidden: make([]float32, ffn),
|
||||||
|
}
|
||||||
|
// One vocabulary of logits per beam, reused at every step: the largest
|
||||||
|
// allocation of the search.
|
||||||
|
probs := make([][]float32, numBeams)
|
||||||
|
for i := range probs {
|
||||||
|
probs[i] = make([]float32, m.cfg.VocabSize)
|
||||||
}
|
}
|
||||||
beams := []beam{{
|
beams := []beam{{
|
||||||
tokens: []int{m.cfg.DecoderStart},
|
tokens: []int{m.cfg.DecoderStart},
|
||||||
keys: make([][]float32, len(m.decoder)),
|
keys: make([][]float32, len(m.decoder)),
|
||||||
values: make([][]float32, len(m.decoder)),
|
values: make([][]float32, len(m.decoder)),
|
||||||
}}
|
}}
|
||||||
maxLen := min(m.cfg.MaxPositions-1, 2*srcLen+10)
|
|
||||||
var done []hypothesis
|
var done []hypothesis
|
||||||
for len(beams[0].tokens) <= maxLen {
|
for len(beams[0].tokens) <= maxLen {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -54,9 +80,9 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
|||||||
}
|
}
|
||||||
hidden := make([][]float32, len(beams))
|
hidden := make([][]float32, len(beams))
|
||||||
for i := range beams {
|
for i := range beams {
|
||||||
hidden[i] = m.decodeStep(&beams[i], crossK, crossV, srcLen)
|
hidden[i] = m.decodeStep(&beams[i], crossK, crossV, srcLen, scratch)
|
||||||
}
|
}
|
||||||
logProbs := m.logProbs(hidden)
|
logProbs := m.logProbs(hidden, probs[:len(beams)])
|
||||||
var cands []candidate
|
var cands []candidate
|
||||||
for i, lp := range logProbs {
|
for i, lp := range logProbs {
|
||||||
for _, t := range topK(lp, 2*numBeams) {
|
for _, t := range topK(lp, 2*numBeams) {
|
||||||
@@ -72,29 +98,45 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
|||||||
}
|
}
|
||||||
return 0
|
return 0
|
||||||
})
|
})
|
||||||
var next []beam
|
// Choose the surviving beams first, then build them: a cache is
|
||||||
|
// copied at most once per step, and only when several survivors
|
||||||
|
// share its parent.
|
||||||
|
children := make([]int, len(beams))
|
||||||
|
chosen := make([]candidate, 0, numBeams)
|
||||||
for rank, c := range cands {
|
for rank, c := range cands {
|
||||||
parent := beams[c.parent]
|
|
||||||
if c.token == m.cfg.EOSToken {
|
if c.token == m.cfg.EOSToken {
|
||||||
if rank < numBeams {
|
if rank < numBeams {
|
||||||
|
parent := beams[c.parent]
|
||||||
done = append(done, hypothesis{tokens: slices.Clone(parent.tokens), score: c.score / float64(len(parent.tokens))})
|
done = append(done, hypothesis{tokens: slices.Clone(parent.tokens), score: c.score / float64(len(parent.tokens))})
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
chosen = append(chosen, c)
|
||||||
|
children[c.parent]++
|
||||||
|
if len(chosen) == numBeams {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next := make([]beam, 0, len(chosen))
|
||||||
|
for _, c := range chosen {
|
||||||
|
parent := &beams[c.parent]
|
||||||
child := beam{
|
child := beam{
|
||||||
tokens: append(slices.Clone(parent.tokens), c.token),
|
tokens: append(slices.Clone(parent.tokens), c.token),
|
||||||
score: c.score,
|
score: c.score,
|
||||||
keys: make([][]float32, len(parent.keys)),
|
keys: make([][]float32, len(parent.keys)),
|
||||||
values: make([][]float32, len(parent.values)),
|
values: make([][]float32, len(parent.values)),
|
||||||
}
|
}
|
||||||
|
takeOver := children[c.parent] == 1
|
||||||
for l := range parent.keys {
|
for l := range parent.keys {
|
||||||
child.keys[l] = slices.Clone(parent.keys[l])
|
if takeOver {
|
||||||
child.values[l] = slices.Clone(parent.values[l])
|
child.keys[l] = parent.keys[l]
|
||||||
|
child.values[l] = parent.values[l]
|
||||||
|
} else {
|
||||||
|
child.keys[l] = slices.Clone(parent.keys[l])
|
||||||
|
child.values[l] = slices.Clone(parent.values[l])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
next = append(next, child)
|
next = append(next, child)
|
||||||
if len(next) == numBeams {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
slices.SortStableFunc(done, func(a, b hypothesis) int {
|
slices.SortStableFunc(done, func(a, b hypothesis) int {
|
||||||
switch {
|
switch {
|
||||||
@@ -126,22 +168,17 @@ func (m *Model) beamSearch(ctx context.Context, enc []float32, srcLen int) ([]in
|
|||||||
|
|
||||||
// decodeStep runs the decoder for the beam's last token and returns its
|
// decodeStep runs the decoder for the beam's last token and returns its
|
||||||
// output vector.
|
// output vector.
|
||||||
func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []float32 {
|
func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int, s *decoderScratch) []float32 {
|
||||||
d := m.cfg.DModel
|
d := m.cfg.DModel
|
||||||
pos := len(b.tokens) - 1
|
pos := len(b.tokens) - 1
|
||||||
tok := b.tokens[pos]
|
tok := b.tokens[pos]
|
||||||
x := make([]float32, d)
|
x := make([]float32, d) // the beam's output vector, which lives past the step
|
||||||
emb := m.embed[tok*d : (tok+1)*d]
|
emb := m.embed[tok*d : (tok+1)*d]
|
||||||
p := m.decPos[pos*d : (pos+1)*d]
|
p := m.decPos[pos*d : (pos+1)*d]
|
||||||
for j := range x {
|
for j := range x {
|
||||||
x[j] = emb[j]*m.embedScale + p[j]
|
x[j] = emb[j]*m.embedScale + p[j]
|
||||||
}
|
}
|
||||||
q := make([]float32, d)
|
q, k, v, ctxv, tmp, scores := s.q, s.k, s.v, s.ctxv, s.tmp, s.scores
|
||||||
k := make([]float32, d)
|
|
||||||
v := make([]float32, d)
|
|
||||||
ctxv := make([]float32, d)
|
|
||||||
tmp := make([]float32, d)
|
|
||||||
scores := make([]float32, max(pos+1, srcLen))
|
|
||||||
for i, l := range m.decoder {
|
for i, l := range m.decoder {
|
||||||
l.self.q.apply(q, x)
|
l.self.q.apply(q, x)
|
||||||
l.self.k.apply(k, x)
|
l.self.k.apply(k, x)
|
||||||
@@ -157,7 +194,7 @@ func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []fl
|
|||||||
l.cross.o.apply(tmp, ctxv)
|
l.cross.o.apply(tmp, ctxv)
|
||||||
addNorm(x, tmp, l.crossNorm, 1)
|
addNorm(x, tmp, l.crossNorm, 1)
|
||||||
|
|
||||||
hidden := make([]float32, l.fc1.out)
|
hidden := s.hidden[:l.fc1.out]
|
||||||
l.fc1.apply(hidden, x)
|
l.fc1.apply(hidden, x)
|
||||||
for j := range hidden {
|
for j := range hidden {
|
||||||
hidden[j] = m.activation(hidden[j])
|
hidden[j] = m.activation(hidden[j])
|
||||||
@@ -169,15 +206,15 @@ func (m *Model) decodeStep(b *beam, crossK, crossV [][]float32, srcLen int) []fl
|
|||||||
}
|
}
|
||||||
|
|
||||||
// logProbs projects each beam's vector on the vocabulary (the embedding
|
// logProbs projects each beam's vector on the vocabulary (the embedding
|
||||||
// matrix, shared with the output layer) and returns log-softmax scores.
|
// matrix, shared with the output layer) and returns log-softmax scores in
|
||||||
// This is the largest product of a step, so it is split across CPUs.
|
// out, one vocabulary per beam, reused from step to step. This is the
|
||||||
func (m *Model) logProbs(hidden [][]float32) [][]float32 {
|
// largest product of a step, so it is split across CPUs.
|
||||||
|
func (m *Model) logProbs(hidden, out [][]float32) [][]float32 {
|
||||||
d, vocab := m.cfg.DModel, m.cfg.VocabSize
|
d, vocab := m.cfg.DModel, m.cfg.VocabSize
|
||||||
out := make([][]float32, len(hidden))
|
|
||||||
for i := range out {
|
|
||||||
out[i] = make([]float32, vocab)
|
|
||||||
}
|
|
||||||
workers := runtime.GOMAXPROCS(0)
|
workers := runtime.GOMAXPROCS(0)
|
||||||
|
if m.threads > 0 {
|
||||||
|
workers = min(workers, m.threads)
|
||||||
|
}
|
||||||
chunk := (vocab + workers - 1) / workers
|
chunk := (vocab + workers - 1) / workers
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
for lo := 0; lo < vocab; lo += chunk {
|
for lo := 0; lo < vocab; lo += chunk {
|
||||||
|
|||||||
+110
-29
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -20,20 +21,12 @@ type RemoteFile struct {
|
|||||||
SHA256 string
|
SHA256 string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Installed reports whether dir holds every file of a model.
|
// Download fetches files into dir. A file already there with the right
|
||||||
func Installed(dir string) bool {
|
// size and digest is skipped. A file is downloaded into a ".partial" file
|
||||||
for _, f := range Files {
|
// that is renamed only once its digest checks out, so an interrupted
|
||||||
if st, err := os.Stat(filepath.Join(dir, f)); err != nil || st.Size() == 0 {
|
// download never leaves a half-written model behind; the partial file is
|
||||||
return false
|
// kept and the next Download resumes it where it stopped. progress
|
||||||
}
|
// receives the bytes downloaded so far and the total.
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download fetches files into dir. Each file is written next to its
|
|
||||||
// final name and renamed once its digest is checked, so an interrupted
|
|
||||||
// download never leaves a half-written model behind. progress receives
|
|
||||||
// the bytes downloaded so far and the total.
|
|
||||||
func Download(ctx context.Context, client *http.Client, dir string, files []RemoteFile, progress func(done, total int64)) error {
|
func Download(ctx context.Context, client *http.Client, dir string, files []RemoteFile, progress func(done, total int64)) error {
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -42,57 +35,145 @@ func Download(ctx context.Context, client *http.Client, dir string, files []Remo
|
|||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
total += f.Size
|
total += f.Size
|
||||||
}
|
}
|
||||||
|
report := func(n int64) {
|
||||||
|
if progress != nil {
|
||||||
|
progress(done+n, total)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
n, err := download(ctx, client, dir, f, func(n int64) {
|
report(0)
|
||||||
if progress != nil {
|
n, err := download(ctx, client, dir, f, report)
|
||||||
progress(done+n, total)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("downloading %s: %w", f.Name, err)
|
return fmt.Errorf("downloading %s: %w", f.Name, err)
|
||||||
}
|
}
|
||||||
done += n
|
done += n
|
||||||
|
report(0)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// download fetches one file, resuming the partial download left by an
|
||||||
|
// interrupted run when the server supports it.
|
||||||
func download(ctx context.Context, client *http.Client, dir string, f RemoteFile, progress func(int64)) (int64, error) {
|
func download(ctx context.Context, client *http.Client, dir string, f RemoteFile, progress func(int64)) (int64, error) {
|
||||||
|
final := filepath.Join(dir, f.Name)
|
||||||
|
if ok, err := verify(final, f); err != nil {
|
||||||
|
return 0, err
|
||||||
|
} else if ok {
|
||||||
|
return f.Size, nil // downloaded by a previous run
|
||||||
|
}
|
||||||
|
partial := filepath.Join(dir, "."+f.Name+".partial")
|
||||||
|
var off int64
|
||||||
|
if st, err := os.Stat(partial); err == nil && st.Size() > 0 && st.Size() <= f.Size {
|
||||||
|
off = st.Size()
|
||||||
|
}
|
||||||
|
if off == f.Size {
|
||||||
|
// A previous run was interrupted between the digest check and
|
||||||
|
// the rename.
|
||||||
|
if ok, _ := verify(partial, f); !ok {
|
||||||
|
off = 0 // corrupt: start over
|
||||||
|
} else if err := os.Rename(partial, final); err != nil {
|
||||||
|
return 0, err
|
||||||
|
} else {
|
||||||
|
return f.Size, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, f.URL, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, f.URL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
if off > 0 {
|
||||||
|
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", off))
|
||||||
|
}
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode != http.StatusOK {
|
if off > 0 && resp.StatusCode == http.StatusOK {
|
||||||
|
off = 0 // the server cannot resume: start over
|
||||||
|
}
|
||||||
|
want := http.StatusOK
|
||||||
|
if off > 0 {
|
||||||
|
want = http.StatusPartialContent
|
||||||
|
}
|
||||||
|
if resp.StatusCode != want {
|
||||||
|
if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
|
||||||
|
os.Remove(partial) // the partial file cannot be resumed
|
||||||
|
}
|
||||||
return 0, fmt.Errorf("server answered %s", resp.Status)
|
return 0, fmt.Errorf("server answered %s", resp.Status)
|
||||||
}
|
}
|
||||||
tmp, err := os.CreateTemp(dir, "."+f.Name+".*")
|
tmp, err := os.OpenFile(partial, os.O_CREATE|os.O_RDWR, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer os.Remove(tmp.Name()) // no-op once renamed
|
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
w := &countingWriter{w: io.MultiWriter(tmp, hash), progress: progress}
|
if off > 0 {
|
||||||
_, err = io.Copy(w, io.LimitReader(resp.Body, f.Size+1))
|
// The digest covers the whole file: seed it with the bytes
|
||||||
|
// already on disk.
|
||||||
|
if _, err := io.CopyN(hash, tmp, off); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return 0, fmt.Errorf("reading the partial download: %w", err)
|
||||||
|
}
|
||||||
|
} else if err := tmp.Truncate(0); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if _, err := tmp.Seek(off, io.SeekStart); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
w := &countingWriter{w: io.MultiWriter(tmp, hash), progress: func(n int64) { progress(off + n) }}
|
||||||
|
_, err = io.Copy(w, io.LimitReader(resp.Body, f.Size-off+1))
|
||||||
if cerr := tmp.Close(); err == nil {
|
if cerr := tmp.Close(); err == nil {
|
||||||
err = cerr
|
err = cerr
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err // the partial file stays, for the next run
|
||||||
}
|
}
|
||||||
if w.n != f.Size {
|
if got := off + w.n; got != f.Size {
|
||||||
return 0, fmt.Errorf("got %d bytes, expected %d", w.n, f.Size)
|
return 0, fmt.Errorf("got %d bytes, expected %d", got, f.Size)
|
||||||
}
|
}
|
||||||
if sum := hex.EncodeToString(hash.Sum(nil)); sum != f.SHA256 {
|
if sum := hex.EncodeToString(hash.Sum(nil)); sum != f.SHA256 {
|
||||||
|
os.Remove(partial) // corrupt: resuming it would fail again
|
||||||
return 0, fmt.Errorf("checksum mismatch")
|
return 0, fmt.Errorf("checksum mismatch")
|
||||||
}
|
}
|
||||||
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
if err := os.Chmod(partial, 0o644); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
return w.n, os.Rename(tmp.Name(), filepath.Join(dir, f.Name))
|
return f.Size, os.Rename(partial, final)
|
||||||
|
}
|
||||||
|
|
||||||
|
// verify reports whether path holds the file, with its expected size and
|
||||||
|
// digest. A missing file is not an error.
|
||||||
|
func verify(path string, f RemoteFile) (bool, error) {
|
||||||
|
st, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if st.Size() != f.Size {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
sum, err := hashFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return sum == f.SHA256, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashFile(path string) (string, error) {
|
||||||
|
r, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(h, r); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type countingWriter struct {
|
type countingWriter struct {
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ func TestSentencePiece(t *testing.T) {
|
|||||||
func TestSafetensors(t *testing.T) {
|
func TestSafetensors(t *testing.T) {
|
||||||
path := filepath.Join(t.TempDir(), "m.safetensors")
|
path := filepath.Join(t.TempDir(), "m.safetensors")
|
||||||
writeSafetensors(t, path, []testTensor{{"w", []int{2, 2}, []float32{1, -2, 3.5, 0}}})
|
writeSafetensors(t, path, []testTensor{{"w", []int{2, 2}, []float32{1, -2, 3.5, 0}}})
|
||||||
got, err := readSafetensors(path)
|
got, err := readSafetensors(context.Background(), path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -141,7 +141,7 @@ func TestSafetensors(t *testing.T) {
|
|||||||
data = append(data, h...)
|
data = append(data, h...)
|
||||||
data = append(data, 0x00, 0x3c, 0x00, 0xc0, 0x80, 0x3f) // 1, -2 (F16), 1 (BF16)
|
data = append(data, 0x00, 0x3c, 0x00, 0xc0, 0x80, 0x3f) // 1, -2 (F16), 1 (BF16)
|
||||||
writeFile(t, path, data)
|
writeFile(t, path, data)
|
||||||
got, err = readSafetensors(path)
|
got, err = readSafetensors(context.Background(), path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -151,7 +151,7 @@ func TestSafetensors(t *testing.T) {
|
|||||||
|
|
||||||
h = []byte(`{"a":{"dtype":"F32","shape":[3],"data_offsets":[0,4]}}`)
|
h = []byte(`{"a":{"dtype":"F32","shape":[3],"data_offsets":[0,4]}}`)
|
||||||
writeFile(t, path, append(binary.LittleEndian.AppendUint64(nil, uint64(len(h))), append(h, 0, 0, 0, 0)...))
|
writeFile(t, path, append(binary.LittleEndian.AppendUint64(nil, uint64(len(h))), append(h, 0, 0, 0, 0)...))
|
||||||
if _, err := readSafetensors(path); err == nil {
|
if _, err := readSafetensors(context.Background(), path); err == nil {
|
||||||
t.Error("shape mismatch accepted")
|
t.Error("shape mismatch accepted")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -251,10 +251,7 @@ func tinyModel(t *testing.T) string {
|
|||||||
|
|
||||||
func TestTinyModel(t *testing.T) {
|
func TestTinyModel(t *testing.T) {
|
||||||
dir := tinyModel(t)
|
dir := tinyModel(t)
|
||||||
if !Installed(dir) {
|
m, err := Load(context.Background(), dir)
|
||||||
t.Fatal("tiny model not installed")
|
|
||||||
}
|
|
||||||
m, err := Load(dir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -281,10 +278,7 @@ func TestTinyModel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
os.Remove(filepath.Join(dir, "vocab.json"))
|
os.Remove(filepath.Join(dir, "vocab.json"))
|
||||||
if Installed(dir) {
|
if _, err := Load(context.Background(), dir); err == nil {
|
||||||
t.Error("incomplete model reported installed")
|
|
||||||
}
|
|
||||||
if _, err := Load(dir); err == nil {
|
|
||||||
t.Error("incomplete model loaded")
|
t.Error("incomplete model loaded")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -297,7 +291,7 @@ func TestRealModel(t *testing.T) {
|
|||||||
if dir == "" {
|
if dir == "" {
|
||||||
t.Skip("PLOTZILLA_MT_MODEL not set")
|
t.Skip("PLOTZILLA_MT_MODEL not set")
|
||||||
}
|
}
|
||||||
m, err := Load(dir)
|
m, err := Load(context.Background(), dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -315,12 +309,23 @@ func TestRealModel(t *testing.T) {
|
|||||||
func TestDownload(t *testing.T) {
|
func TestDownload(t *testing.T) {
|
||||||
content := []byte("model weights")
|
content := []byte("model weights")
|
||||||
sum := sha256.Sum256(content)
|
sum := sha256.Sum256(content)
|
||||||
|
var requests int
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path == "/missing" {
|
requests++
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/missing":
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
return
|
case "/full": // a server without resume support
|
||||||
|
w.Write(content)
|
||||||
|
default:
|
||||||
|
var off int
|
||||||
|
if _, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-", &off); err == nil && off > 0 && off < len(content) {
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
w.Write(content[off:])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Write(content)
|
||||||
}
|
}
|
||||||
w.Write(content)
|
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
dir := filepath.Join(t.TempDir(), "model")
|
dir := filepath.Join(t.TempDir(), "model")
|
||||||
@@ -334,6 +339,52 @@ func TestDownload(t *testing.T) {
|
|||||||
t.Errorf("downloaded %q, progress %d", got, last)
|
t.Errorf("downloaded %q, progress %d", got, last)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A file already downloaded and verified is skipped.
|
||||||
|
if err := Download(context.Background(), srv.Client(), dir, []RemoteFile{file}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if requests != 1 {
|
||||||
|
t.Errorf("a complete file was downloaded again (%d requests)", requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An interrupted download resumes its partial file.
|
||||||
|
resumed := file
|
||||||
|
resumed.Name, resumed.URL = "r.bin", srv.URL+"/r"
|
||||||
|
writeFile(t, filepath.Join(dir, ".r.bin.partial"), content[:6])
|
||||||
|
if err := Download(context.Background(), srv.Client(), dir, []RemoteFile{resumed}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(filepath.Join(dir, "r.bin")); string(got) != string(content) {
|
||||||
|
t.Errorf("resumed download = %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A server that cannot resume answers the whole file: the partial
|
||||||
|
// download is restarted.
|
||||||
|
full := file
|
||||||
|
full.Name, full.URL = "full.bin", srv.URL+"/full"
|
||||||
|
writeFile(t, filepath.Join(dir, ".full.bin.partial"), content[:6])
|
||||||
|
if err := Download(context.Background(), srv.Client(), dir, []RemoteFile{full}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(filepath.Join(dir, "full.bin")); string(got) != string(content) {
|
||||||
|
t.Errorf("restarted download = %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A corrupt partial file fails the digest check and is removed: the
|
||||||
|
// next download starts from scratch.
|
||||||
|
corrupt := file
|
||||||
|
corrupt.Name, corrupt.URL = "c.bin", srv.URL+"/c"
|
||||||
|
writeFile(t, filepath.Join(dir, ".c.bin.partial"), []byte("garbag"))
|
||||||
|
if err := Download(context.Background(), srv.Client(), dir, []RemoteFile{corrupt}, nil); err == nil {
|
||||||
|
t.Error("corrupt partial file accepted")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, ".c.bin.partial")); err == nil {
|
||||||
|
t.Error("corrupt partial file kept")
|
||||||
|
}
|
||||||
|
if err := Download(context.Background(), srv.Client(), dir, []RemoteFile{corrupt}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
bad := file
|
bad := file
|
||||||
bad.Name, bad.SHA256 = "bad.bin", strings.Repeat("0", 64)
|
bad.Name, bad.SHA256 = "bad.bin", strings.Repeat("0", 64)
|
||||||
missing := file
|
missing := file
|
||||||
@@ -348,8 +399,13 @@ func TestDownload(t *testing.T) {
|
|||||||
t.Errorf("%s: a failed download left the file", f.Name)
|
t.Errorf("%s: a failed download left the file", f.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entries, _ := os.ReadDir(dir)
|
// Only the interrupted download keeps a partial file, to be resumed.
|
||||||
if len(entries) != 1 {
|
for _, f := range []RemoteFile{bad, missing} {
|
||||||
t.Errorf("temporary files left: %v", entries)
|
if _, err := os.Stat(filepath.Join(dir, "."+f.Name+".partial")); err == nil {
|
||||||
|
t.Errorf("%s: a failed download left a partial file", f.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, ".short.bin.partial")); err != nil {
|
||||||
|
t.Error("an interrupted download left no partial file to resume")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+43
-13
@@ -3,6 +3,18 @@
|
|||||||
// plain Go: a SentencePiece tokenizer, the transformer encoder-decoder and
|
// plain Go: a SentencePiece tokenizer, the transformer encoder-decoder and
|
||||||
// beam search. A model directory holds config.json, model.safetensors,
|
// beam search. A model directory holds config.json, model.safetensors,
|
||||||
// source.spm, target.spm and vocab.json.
|
// source.spm, target.spm and vocab.json.
|
||||||
|
//
|
||||||
|
// The engine lives in-tree on purpose. Plotzilla ships as one binary built
|
||||||
|
// with only the Go toolchain: a native runtime (ctranslate2, argos) would
|
||||||
|
// add cgo and per-distribution packaging, and a separate binary or plugin
|
||||||
|
// an install step, all for a single menu item. Translation is offline by
|
||||||
|
// design — an unpublished script never leaves the machine — and the model
|
||||||
|
// is data downloaded at runtime (Apache-2.0), so neither the "no
|
||||||
|
// third-party dependency" rule nor the GPL is affected. The screenplay
|
||||||
|
// conventions stay out of this package (internal/translate owns them) and
|
||||||
|
// the pipeline is covered by a synthetic tiny model and, on demand, the
|
||||||
|
// real one. If more language pairs are ever added, each with its own
|
||||||
|
// 300 MB download and quirks, reconsider a separate optional component.
|
||||||
package marian
|
package marian
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -14,9 +26,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Files are the files a model directory must contain.
|
|
||||||
var Files = []string{"config.json", "model.safetensors", "source.spm", "target.spm", "vocab.json"}
|
|
||||||
|
|
||||||
type config struct {
|
type config struct {
|
||||||
DModel int `json:"d_model"`
|
DModel int `json:"d_model"`
|
||||||
EncoderLayers int `json:"encoder_layers"`
|
EncoderLayers int `json:"encoder_layers"`
|
||||||
@@ -60,10 +69,12 @@ type decoderLayer struct {
|
|||||||
finalLayerNorm layerNorm
|
finalLayerNorm layerNorm
|
||||||
}
|
}
|
||||||
|
|
||||||
// Model is a loaded translation model. It is read-only once loaded:
|
// Model is a loaded translation model. It is read-only once loaded (but
|
||||||
// Translate may be called from several goroutines at once.
|
// for SetThreads, right after Load): Translate may be called from several
|
||||||
|
// goroutines at once.
|
||||||
type Model struct {
|
type Model struct {
|
||||||
cfg config
|
cfg config
|
||||||
|
threads int // goroutines a decoding step may start (0: one per CPU)
|
||||||
embed []float32 // vocab × d, shared by encoder, decoder and output
|
embed []float32 // vocab × d, shared by encoder, decoder and output
|
||||||
encPos []float32 // positions × d
|
encPos []float32 // positions × d
|
||||||
decPos []float32
|
decPos []float32
|
||||||
@@ -79,8 +90,12 @@ type Model struct {
|
|||||||
activation func(float32) float32
|
activation func(float32) float32
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load reads the model in dir.
|
// Load reads the model in dir. The load can take seconds for a large
|
||||||
func Load(dir string) (*Model, error) {
|
// model: it stops early when ctx is cancelled.
|
||||||
|
func Load(ctx context.Context, dir string) (*Model, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
var cfg config
|
var cfg config
|
||||||
data, err := os.ReadFile(filepath.Join(dir, "config.json"))
|
data, err := os.ReadFile(filepath.Join(dir, "config.json"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -130,7 +145,10 @@ func Load(dir string) (*Model, error) {
|
|||||||
}
|
}
|
||||||
m.unk = m.ids["<unk>"]
|
m.unk = m.ids["<unk>"]
|
||||||
|
|
||||||
tensors, err := readSafetensors(filepath.Join(dir, "model.safetensors"))
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tensors, err := readSafetensors(ctx, filepath.Join(dir, "model.safetensors"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading model.safetensors: %w", err)
|
return nil, fmt.Errorf("reading model.safetensors: %w", err)
|
||||||
}
|
}
|
||||||
@@ -214,6 +232,13 @@ func (m *Model) bind(t map[string]tensor) error {
|
|||||||
return missing
|
return missing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetThreads caps how many goroutines each decoding step starts (0: one
|
||||||
|
// per CPU, the default). When several sentences are translated at once,
|
||||||
|
// give each its share of the CPUs, or every step of every sentence starts
|
||||||
|
// a team of goroutines and they oversubscribe the machine. It must be
|
||||||
|
// called before the first Translate.
|
||||||
|
func (m *Model) SetThreads(n int) { m.threads = n }
|
||||||
|
|
||||||
// maxSourceTokens leaves room for the end-of-sentence token.
|
// maxSourceTokens leaves room for the end-of-sentence token.
|
||||||
func (m *Model) maxSourceTokens() int { return m.cfg.MaxPositions - 1 }
|
func (m *Model) maxSourceTokens() int { return m.cfg.MaxPositions - 1 }
|
||||||
|
|
||||||
@@ -269,6 +294,11 @@ func (m *Model) encode(ids []int) []float32 {
|
|||||||
ctxv := make([]float32, n*d)
|
ctxv := make([]float32, n*d)
|
||||||
tmp := make([]float32, n*d)
|
tmp := make([]float32, n*d)
|
||||||
scores := make([]float32, n)
|
scores := make([]float32, n)
|
||||||
|
ffn := 0 // the hidden layer of every encoder layer, allocated once
|
||||||
|
for _, l := range m.encoder {
|
||||||
|
ffn = max(ffn, l.fc1.out)
|
||||||
|
}
|
||||||
|
hidden := make([]float32, n*ffn)
|
||||||
for _, l := range m.encoder {
|
for _, l := range m.encoder {
|
||||||
l.self.q.rows(q, x, n)
|
l.self.q.rows(q, x, n)
|
||||||
l.self.k.rows(k, x, n)
|
l.self.k.rows(k, x, n)
|
||||||
@@ -278,12 +308,12 @@ func (m *Model) encode(ids []int) []float32 {
|
|||||||
}
|
}
|
||||||
l.self.o.rows(tmp, ctxv, n)
|
l.self.o.rows(tmp, ctxv, n)
|
||||||
addNorm(x, tmp, l.selfNorm, n)
|
addNorm(x, tmp, l.selfNorm, n)
|
||||||
hidden := make([]float32, n*l.fc1.out)
|
h := hidden[:n*l.fc1.out]
|
||||||
l.fc1.rows(hidden, x, n)
|
l.fc1.rows(h, x, n)
|
||||||
for i := range hidden {
|
for i := range h {
|
||||||
hidden[i] = m.activation(hidden[i])
|
h[i] = m.activation(h[i])
|
||||||
}
|
}
|
||||||
l.fc2.rows(tmp, hidden, n)
|
l.fc2.rows(tmp, h, n)
|
||||||
addNorm(x, tmp, l.finalLayerNorm, n)
|
addNorm(x, tmp, l.finalLayerNorm, n)
|
||||||
}
|
}
|
||||||
return x
|
return x
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package marian
|
package marian
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -27,7 +28,7 @@ type tensorInfo struct {
|
|||||||
// little-endian header length, a JSON header, then the raw data) as
|
// little-endian header length, a JSON header, then the raw data) as
|
||||||
// float32. F32 tensors are read straight into their slice: the file is
|
// float32. F32 tensors are read straight into their slice: the file is
|
||||||
// large and a second copy would double the memory needed to load it.
|
// large and a second copy would double the memory needed to load it.
|
||||||
func readSafetensors(path string) (map[string]tensor, error) {
|
func readSafetensors(ctx context.Context, path string) (map[string]tensor, error) {
|
||||||
f, err := os.Open(path)
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -54,6 +55,9 @@ func readSafetensors(path string) (map[string]tensor, error) {
|
|||||||
if name == "__metadata__" {
|
if name == "__metadata__" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
var info tensorInfo
|
var info tensorInfo
|
||||||
if err := json.Unmarshal(raw, &info); err != nil {
|
if err := json.Unmarshal(raw, &info); err != nil {
|
||||||
return nil, fmt.Errorf("tensor %s: %w", name, err)
|
return nil, fmt.Errorf("tensor %s: %w", name, err)
|
||||||
|
|||||||
+44
-24
@@ -9,6 +9,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode/utf16"
|
"unicode/utf16"
|
||||||
|
|
||||||
@@ -34,8 +35,11 @@ const (
|
|||||||
var fontNames = []string{"Courier", "Courier-Bold", "Courier-Oblique", "Courier-BoldOblique"}
|
var fontNames = []string{"Courier", "Courier-Bold", "Courier-Oblique", "Courier-BoldOblique"}
|
||||||
|
|
||||||
// Render writes pages to w. The title page, if any, must be the first page
|
// Render writes pages to w. The title page, if any, must be the first page
|
||||||
// and have Number 0 so that it is not numbered.
|
// and have Number 0 so that it is not numbered. It returns, sorted, the
|
||||||
func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
// characters the standard Courier fonts cannot draw: each one is written
|
||||||
|
// as "?" in the output, so the caller can warn instead of corrupting the
|
||||||
|
// script silently.
|
||||||
|
func Render(w io.Writer, pages []layout.Page, opt Options) ([]rune, error) {
|
||||||
geo := layout.PaperGeometry(opt.Paper)
|
geo := layout.PaperGeometry(opt.Paper)
|
||||||
var objs [][]byte // objs[i] is object number i+1
|
var objs [][]byte // objs[i] is object number i+1
|
||||||
add := func(s string) int {
|
add := func(s string) int {
|
||||||
@@ -50,11 +54,12 @@ func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
|||||||
}
|
}
|
||||||
add(fmt.Sprintf("<< /Title %s /Author %s /Creator (Plotzilla) /Producer (Plotzilla) >>", textString(opt.Title), textString(opt.Author)))
|
add(fmt.Sprintf("<< /Title %s /Author %s /Creator (Plotzilla) /Producer (Plotzilla) >>", textString(opt.Title), textString(opt.Author)))
|
||||||
|
|
||||||
|
missing := map[rune]bool{}
|
||||||
var kids []string
|
var kids []string
|
||||||
for _, pg := range pages {
|
for _, pg := range pages {
|
||||||
stream, err := deflate(pageContent(pg, geo, opt.Revisions))
|
stream, err := deflate(pageContent(pg, geo, opt.Revisions, missing))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
content := add(fmt.Sprintf("<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream", len(stream), stream))
|
content := add(fmt.Sprintf("<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream", len(stream), stream))
|
||||||
page := add(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %g %g] "+
|
page := add(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %g %g] "+
|
||||||
@@ -80,7 +85,16 @@ func Render(w io.Writer, pages []layout.Page, opt Options) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R /Info 7 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objs)+1, xref)
|
fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R /Info 7 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objs)+1, xref)
|
||||||
_, err := w.Write(buf.Bytes())
|
_, err := w.Write(buf.Bytes())
|
||||||
return err
|
return sortedRunes(missing), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedRunes(set map[rune]bool) []rune {
|
||||||
|
out := make([]rune, 0, len(set))
|
||||||
|
for r := range set {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
slices.Sort(out)
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// points converts inches to PDF points, rounded to 1/100 pt.
|
// points converts inches to PDF points, rounded to 1/100 pt.
|
||||||
@@ -92,16 +106,16 @@ func baseline(geo layout.Geometry, row int) float64 {
|
|||||||
return top - float64(row)*fontSize - 9
|
return top - float64(row)*fontSize - 9
|
||||||
}
|
}
|
||||||
|
|
||||||
func text(b *bytes.Buffer, x, y float64, s string) {
|
func text(b *bytes.Buffer, x, y float64, s string, missing map[rune]bool) {
|
||||||
fmt.Fprintf(b, "BT /F1 12 Tf %.2f %.2f Td %s Tj ET\n", x, y, pdfString(s))
|
fmt.Fprintf(b, "BT /F1 12 Tf %.2f %.2f Td %s Tj ET\n", x, y, pdfString(s, missing))
|
||||||
}
|
}
|
||||||
|
|
||||||
func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Revision) []byte {
|
func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Revision, missing map[rune]bool) []byte {
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
header := (geo.Height-0.5)*ptPerIn - 9
|
header := (geo.Height-0.5)*ptPerIn - 9
|
||||||
if pg.Number > 0 && pg.Label != "" && pg.Label != "1" {
|
if pg.Number > 0 && pg.Label != "" && pg.Label != "1" {
|
||||||
num := pg.Label + "."
|
num := pg.Label + "."
|
||||||
text(&b, layout.RightEdge*ptPerIn-float64(len([]rune(num)))*fontSize*0.6, header, num)
|
text(&b, layout.RightEdge*ptPerIn-float64(len([]rune(num)))*fontSize*0.6, header, num, missing)
|
||||||
}
|
}
|
||||||
// A revised page names its revision; its latest changes are starred.
|
// A revised page names its revision; its latest changes are starred.
|
||||||
rev := pg.Rev()
|
rev := pg.Rev()
|
||||||
@@ -114,17 +128,17 @@ func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Rev
|
|||||||
if r.Date != "" {
|
if r.Date != "" {
|
||||||
label += " - " + r.Date
|
label += " - " + r.Date
|
||||||
}
|
}
|
||||||
text(&b, 1.5*ptPerIn, header, label)
|
text(&b, 1.5*ptPerIn, header, label, missing)
|
||||||
}
|
}
|
||||||
for _, ln := range pg.Lines {
|
for _, ln := range pg.Lines {
|
||||||
y := baseline(geo, ln.Row)
|
y := baseline(geo, ln.Row)
|
||||||
x := ln.X * ptPerIn
|
x := ln.X * ptPerIn
|
||||||
if ln.SceneNumber != "" {
|
if ln.SceneNumber != "" {
|
||||||
text(&b, 0.75*ptPerIn, y, ln.SceneNumber)
|
text(&b, 0.75*ptPerIn, y, ln.SceneNumber, missing)
|
||||||
text(&b, geo.SceneNumber*ptPerIn, y, ln.SceneNumber)
|
text(&b, geo.SceneNumber*ptPerIn, y, ln.SceneNumber, missing)
|
||||||
}
|
}
|
||||||
if rev > 0 && ln.Rev == rev {
|
if rev > 0 && ln.Rev == rev {
|
||||||
text(&b, geo.RevisionMark*ptPerIn, y, "*")
|
text(&b, geo.RevisionMark*ptPerIn, y, "*", missing)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(&b, "BT %.2f %.2f Td ", x, y)
|
fmt.Fprintf(&b, "BT %.2f %.2f Td ", x, y)
|
||||||
cursor := x
|
cursor := x
|
||||||
@@ -142,7 +156,7 @@ func pageContent(pg layout.Page, geo layout.Geometry, revisions []screenplay.Rev
|
|||||||
underlines = append(underlines, [2]float64{cursor, cursor + width})
|
underlines = append(underlines, [2]float64{cursor, cursor + width})
|
||||||
}
|
}
|
||||||
cursor += width
|
cursor += width
|
||||||
fmt.Fprintf(&b, "/F%d 12 Tf %s Tj ", font, pdfString(s.Text))
|
fmt.Fprintf(&b, "/F%d 12 Tf %s Tj ", font, pdfString(s.Text, missing))
|
||||||
}
|
}
|
||||||
b.WriteString("ET\n")
|
b.WriteString("ET\n")
|
||||||
for _, u := range underlines {
|
for _, u := range underlines {
|
||||||
@@ -164,12 +178,16 @@ func deflate(data []byte) ([]byte, error) {
|
|||||||
return b.Bytes(), nil
|
return b.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// pdfString encodes s as a WinAnsi literal string for the Courier fonts.
|
// pdfString encodes s as a WinAnsi literal string for the Courier fonts,
|
||||||
func pdfString(s string) string {
|
// recording in missing the characters they cannot draw.
|
||||||
|
func pdfString(s string, missing map[rune]bool) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
b.WriteByte('(')
|
b.WriteByte('(')
|
||||||
for _, r := range s {
|
for _, r := range s {
|
||||||
c := winAnsi(r)
|
c, ok := winAnsi(r)
|
||||||
|
if !ok {
|
||||||
|
missing[r] = true
|
||||||
|
}
|
||||||
if c == '(' || c == ')' || c == '\\' {
|
if c == '(' || c == ')' || c == '\\' {
|
||||||
b.WriteByte('\\')
|
b.WriteByte('\\')
|
||||||
}
|
}
|
||||||
@@ -189,7 +207,7 @@ func textString(s string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ascii {
|
if ascii {
|
||||||
return pdfString(s)
|
return pdfString(s, map[rune]bool{}) // ASCII only: nothing to record
|
||||||
}
|
}
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
b.WriteString("<FEFF")
|
b.WriteString("<FEFF")
|
||||||
@@ -207,18 +225,20 @@ var cp1252 = map[rune]byte{
|
|||||||
'˜': 0x98, '™': 0x99, 'š': 0x9a, '›': 0x9b, 'œ': 0x9c, 'ž': 0x9e, 'Ÿ': 0x9f,
|
'˜': 0x98, '™': 0x99, 'š': 0x9a, '›': 0x9b, 'œ': 0x9c, 'ž': 0x9e, 'Ÿ': 0x9f,
|
||||||
}
|
}
|
||||||
|
|
||||||
func winAnsi(r rune) byte {
|
// winAnsi encodes r as a WinAnsi byte; ok is false when the standard
|
||||||
|
// fonts cannot draw r and '?' is written instead.
|
||||||
|
func winAnsi(r rune) (c byte, ok bool) {
|
||||||
switch {
|
switch {
|
||||||
case r >= 0x20 && r < 0x7f:
|
case r >= 0x20 && r < 0x7f:
|
||||||
return byte(r)
|
return byte(r), true
|
||||||
case r >= 0xa0 && r <= 0xff:
|
case r >= 0xa0 && r <= 0xff:
|
||||||
return byte(r)
|
return byte(r), true
|
||||||
}
|
}
|
||||||
if c, ok := cp1252[r]; ok {
|
if c, ok := cp1252[r]; ok {
|
||||||
return c
|
return c, true
|
||||||
}
|
}
|
||||||
if r == ' ' || r == ' ' {
|
if r == ' ' || r == ' ' {
|
||||||
return ' ' // narrow spaces used before French punctuation
|
return ' ', true // narrow spaces used before French punctuation
|
||||||
}
|
}
|
||||||
return '?'
|
return '?', false
|
||||||
}
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"compress/zlib"
|
"compress/zlib"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -39,9 +40,13 @@ func TestRender(t *testing.T) {
|
|||||||
pages := []layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)}
|
pages := []layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)}
|
||||||
pages = append(pages, layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
pages = append(pages, layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := Render(&buf, pages, Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
replaced, err := Render(&buf, pages, Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author})
|
||||||
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
if len(replaced) != 0 {
|
||||||
|
t.Errorf("the French sample is fully representable, replaced = %q", string(replaced))
|
||||||
|
}
|
||||||
out := buf.String()
|
out := buf.String()
|
||||||
if !strings.HasPrefix(out, "%PDF-1.4") || !strings.HasSuffix(out, "%%EOF\n") || !strings.Contains(out, "/Count 2") {
|
if !strings.HasPrefix(out, "%PDF-1.4") || !strings.HasSuffix(out, "%%EOF\n") || !strings.Contains(out, "/Count 2") {
|
||||||
t.Fatalf("malformed PDF:\n%.300s", out)
|
t.Fatalf("malformed PDF:\n%.300s", out)
|
||||||
@@ -61,7 +66,7 @@ func TestRevisionsAndA4(t *testing.T) {
|
|||||||
doc.Elements[1].Runs[0].Rev = 1
|
doc.Elements[1].Runs[0].Rev = 1
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
pages := layout.Paginate(doc, layout.Options{}).Pages
|
pages := layout.Paginate(doc, layout.Options{}).Pages
|
||||||
if err := Render(&buf, pages, Options{Paper: doc.Settings.Paper, Revisions: doc.Settings.Revisions}); err != nil {
|
if _, err := Render(&buf, pages, Options{Paper: doc.Settings.Paper, Revisions: doc.Settings.Revisions}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
stream := uncompressed(t, buf.Bytes())
|
stream := uncompressed(t, buf.Bytes())
|
||||||
@@ -74,7 +79,7 @@ func TestRevisionsAndA4(t *testing.T) {
|
|||||||
|
|
||||||
// Without revisions in the options, nothing is marked.
|
// Without revisions in the options, nothing is marked.
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
if err := Render(&buf, pages, Options{Paper: doc.Settings.Paper}); err != nil {
|
if _, err := Render(&buf, pages, Options{Paper: doc.Settings.Paper}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if stream := uncompressed(t, buf.Bytes()); strings.Contains(stream, "Revision") || strings.Contains(stream, "(*)") {
|
if stream := uncompressed(t, buf.Bytes()); strings.Contains(stream, "Revision") || strings.Contains(stream, "(*)") {
|
||||||
@@ -82,6 +87,34 @@ func TestRevisionsAndA4(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestReplacedChars: characters outside WinAnsi are written as '?' and
|
||||||
|
// reported, so an export can warn instead of corrupting the text silently.
|
||||||
|
func TestReplacedChars(t *testing.T) {
|
||||||
|
src := "INT. ΚΑΦΕΝΕΙΟ - ΜΕΡΑ\n\nΚαλημέρα… Plain ASCII.\n"
|
||||||
|
doc := fountain.Parse(src)
|
||||||
|
pages := layout.Paginate(doc, layout.Options{}).Pages
|
||||||
|
var buf bytes.Buffer
|
||||||
|
replaced, err := Render(&buf, pages, Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := string(replaced)
|
||||||
|
for _, r := range src { // every Greek letter is reported
|
||||||
|
if r > 0x7f && r != '…' && !strings.ContainsRune(got, r) {
|
||||||
|
t.Errorf("%q not reported in %q", r, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.ContainsRune(got, '…') { // WinAnsi has "…"
|
||||||
|
t.Errorf("representable … reported in %q", got)
|
||||||
|
}
|
||||||
|
if !slices.IsSorted(replaced) {
|
||||||
|
t.Errorf("replaced not sorted: %q", got)
|
||||||
|
}
|
||||||
|
if stream := uncompressed(t, buf.Bytes()); !strings.Contains(stream, "????????") {
|
||||||
|
t.Errorf("Greek not written as '?':\n%s", stream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// uncompressed returns the content streams of a PDF written by Render.
|
// uncompressed returns the content streams of a PDF written by Render.
|
||||||
func uncompressed(t *testing.T, pdf []byte) string {
|
func uncompressed(t *testing.T, pdf []byte) string {
|
||||||
var out strings.Builder
|
var out strings.Builder
|
||||||
|
|||||||
@@ -90,6 +90,12 @@ func (c *Checker) Add(word string) {
|
|||||||
|
|
||||||
func normalize(w string) string { return strings.ReplaceAll(w, "’", "'") }
|
func normalize(w string) string { return strings.ReplaceAll(w, "’", "'") }
|
||||||
|
|
||||||
|
// cacheLimit bounds the remembered answers, which are keyed on user input:
|
||||||
|
// a long session of typos would grow the map forever otherwise. When full,
|
||||||
|
// the cache is cleared and refilled; a script repeats the same words, so
|
||||||
|
// re-checks are cheap.
|
||||||
|
const cacheLimit = 10000
|
||||||
|
|
||||||
// Check reports whether word is correct. Words with digits ("3rd", "R2D2")
|
// Check reports whether word is correct. Words with digits ("3rd", "R2D2")
|
||||||
// and single letters are always accepted.
|
// and single letters are always accepted.
|
||||||
func (c *Checker) Check(word string) bool {
|
func (c *Checker) Check(word string) bool {
|
||||||
@@ -104,6 +110,9 @@ func (c *Checker) Check(word string) bool {
|
|||||||
}
|
}
|
||||||
ok, found := c.cache[word]
|
ok, found := c.cache[word]
|
||||||
if !found {
|
if !found {
|
||||||
|
if len(c.cache) >= cacheLimit {
|
||||||
|
clear(c.cache)
|
||||||
|
}
|
||||||
ok = c.dict.Check(word)
|
ok = c.dict.Check(word)
|
||||||
c.cache[word] = ok
|
c.cache[word] = ok
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -180,6 +180,35 @@ func TestLatin1(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCheckerCacheBound(t *testing.T) {
|
||||||
|
d, err := Parse([]byte("SET UTF-8\n"), []byte("1\nknown\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c := NewChecker(d, nil)
|
||||||
|
// letters encodes n in bijective base 26: distinct pure-letter words.
|
||||||
|
letters := func(n int) string {
|
||||||
|
var w []rune
|
||||||
|
for {
|
||||||
|
w = append([]rune{rune('a' + n%26)}, w...)
|
||||||
|
if n < 26 {
|
||||||
|
return string(w)
|
||||||
|
}
|
||||||
|
n = n/26 - 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range 2 * cacheLimit {
|
||||||
|
c.Check(letters(i))
|
||||||
|
}
|
||||||
|
if len(c.cache) > cacheLimit {
|
||||||
|
t.Errorf("cache holds %d words, limit %d", len(c.cache), cacheLimit)
|
||||||
|
}
|
||||||
|
// The answers are unchanged by a clear.
|
||||||
|
if !c.Check("known") || c.Check("knownn") {
|
||||||
|
t.Error("wrong answers after the cache cleared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWords(t *testing.T) {
|
func TestWords(t *testing.T) {
|
||||||
got := Words("Élise's well-known R2D2, l’homme… 'quoted' x")
|
got := Words("Élise's well-known R2D2, l’homme… 'quoted' x")
|
||||||
want := []Word{
|
want := []Word{
|
||||||
|
|||||||
@@ -29,9 +29,7 @@ func main() {
|
|||||||
app := NewApp()
|
app := NewApp()
|
||||||
for _, arg := range os.Args[1:] {
|
for _, arg := range os.Args[1:] {
|
||||||
if !strings.HasPrefix(arg, "-") {
|
if !strings.HasPrefix(arg, "-") {
|
||||||
if _, err := os.Stat(arg); err == nil {
|
app.initialPath = arg // may not exist yet: then it is a new script
|
||||||
app.initialPath = arg
|
|
||||||
}
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -49,7 +47,7 @@ func main() {
|
|||||||
BackgroundColour: &options.RGBA{R: 0, G: 0, B: 0, A: 0},
|
BackgroundColour: &options.RGBA{R: 0, G: 0, B: 0, A: 0},
|
||||||
OnStartup: app.startup,
|
OnStartup: app.startup,
|
||||||
OnBeforeClose: app.beforeClose,
|
OnBeforeClose: app.beforeClose,
|
||||||
Bind: []interface{}{app},
|
Bind: []any{app},
|
||||||
Linux: &linux.Options{
|
Linux: &linux.Options{
|
||||||
ProgramName: "plotzilla",
|
ProgramName: "plotzilla",
|
||||||
Icon: icon,
|
Icon: icon,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Sandro Cazzaniga <sandro@cazzaniga.fr>
|
# Maintainer: Sandro Cazzaniga <sandro@cazzaniga.fr>
|
||||||
|
|
||||||
pkgname=plotzilla
|
pkgname=plotzilla
|
||||||
pkgver=1.2
|
pkgver=1.4.1
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc='Screenwriting app for Linux: scripts in the Hollywood standard format, Fountain, FDX and PDF'
|
pkgdesc='Screenwriting app for Linux: scripts in the Hollywood standard format, Fountain, FDX and PDF'
|
||||||
arch=('x86_64' 'aarch64')
|
arch=('x86_64' 'aarch64')
|
||||||
|
|||||||
@@ -277,7 +277,7 @@ func pdfOutput(src, tmp, path string) error {
|
|||||||
pages := append([]layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)},
|
pages := append([]layout.Page{layout.TitlePage(doc.TitlePage, doc.Settings.Paper)},
|
||||||
layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
layout.Paginate(doc, layout.Options{SceneNumbers: true}).Pages...)
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
if err := pdf.Render(&b, pages, pdf.Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
if _, err := pdf.Render(&b, pages, pdf.Options{Title: doc.TitlePage.Title, Author: doc.TitlePage.Author}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
file := filepath.Join(tmp, "sample.pdf")
|
file := filepath.Join(tmp, "sample.pdf")
|
||||||
|
|||||||
+145
-53
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -28,8 +29,14 @@ const (
|
|||||||
// modelIdle is how long the model (300 MB of memory) stays loaded
|
// modelIdle is how long the model (300 MB of memory) stays loaded
|
||||||
// after a translation, ready for the next one.
|
// after a translation, ready for the next one.
|
||||||
modelIdle = 5 * time.Minute
|
modelIdle = 5 * time.Minute
|
||||||
|
// stallTimeout is how long a download may go without receiving a byte
|
||||||
|
// (or its response headers) before it is aborted as stalled.
|
||||||
|
stallTimeout = 30 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// errStalled aborts a download that receives nothing for stallTimeout.
|
||||||
|
var errStalled = fmt.Errorf("the download stalled (no data received for %s)", stallTimeout)
|
||||||
|
|
||||||
var modelFiles = []marian.RemoteFile{
|
var modelFiles = []marian.RemoteFile{
|
||||||
{Name: "config.json", Size: 1416, SHA256: "b3be13d046d9899d7aab8cf4ed624d9a79f5776038ba793f6b4d2ce3e02192f7"},
|
{Name: "config.json", Size: 1416, SHA256: "b3be13d046d9899d7aab8cf4ed624d9a79f5776038ba793f6b4d2ce3e02192f7"},
|
||||||
{Name: "source.spm", Size: 802397, SHA256: "78d0e717c77053f1c4b856d8661d9cb87c64f083a35418c087b9146300e4f585"},
|
{Name: "source.spm", Size: 802397, SHA256: "78d0e717c77053f1c4b856d8661d9cb87c64f083a35418c087b9146300e4f585"},
|
||||||
@@ -46,6 +53,7 @@ type translation struct {
|
|||||||
lexicon translate.Lexicon
|
lexicon translate.Lexicon
|
||||||
release *time.Timer
|
release *time.Timer
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
|
done chan struct{} // closed when the running task is over
|
||||||
}
|
}
|
||||||
|
|
||||||
// TranslationStatus says whether the model is installed, and how much a
|
// TranslationStatus says whether the model is installed, and how much a
|
||||||
@@ -72,15 +80,29 @@ func modelDirs() []string {
|
|||||||
return dirs
|
return dirs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// installedModel returns the first model directory holding every file at
|
||||||
|
// its pinned size. Checking the digests would mean reading 300 MB on
|
||||||
|
// every call; the sizes catch truncated or stale files, and the digests
|
||||||
|
// were checked when the files were downloaded.
|
||||||
func installedModel() string {
|
func installedModel() string {
|
||||||
for _, d := range modelDirs() {
|
for _, d := range modelDirs() {
|
||||||
if marian.Installed(d) {
|
if modelComplete(d) {
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func modelComplete(dir string) bool {
|
||||||
|
for _, f := range modelFiles {
|
||||||
|
st, err := os.Stat(filepath.Join(dir, f.Name))
|
||||||
|
if err != nil || st.Size() != f.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (a *App) TranslationStatus() TranslationStatus {
|
func (a *App) TranslationStatus() TranslationStatus {
|
||||||
var size int64
|
var size int64
|
||||||
for _, f := range modelFiles {
|
for _, f := range modelFiles {
|
||||||
@@ -94,10 +116,11 @@ func (a *App) startTask() (context.Context, error) {
|
|||||||
a.tr.mu.Lock()
|
a.tr.mu.Lock()
|
||||||
defer a.tr.mu.Unlock()
|
defer a.tr.mu.Unlock()
|
||||||
if a.tr.cancel != nil {
|
if a.tr.cancel != nil {
|
||||||
return nil, errors.New("a translation is already running")
|
return nil, errors.New("a translation task is already running")
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithCancel(a.ctx)
|
ctx, cancel := context.WithCancel(a.ctx)
|
||||||
a.tr.cancel = cancel
|
a.tr.cancel = cancel
|
||||||
|
a.tr.done = make(chan struct{})
|
||||||
return ctx, nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,16 +130,22 @@ func (a *App) endTask() {
|
|||||||
if a.tr.cancel != nil {
|
if a.tr.cancel != nil {
|
||||||
a.tr.cancel()
|
a.tr.cancel()
|
||||||
a.tr.cancel = nil
|
a.tr.cancel = nil
|
||||||
|
close(a.tr.done)
|
||||||
|
a.tr.done = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CancelTranslation stops the running download or translation.
|
// CancelTranslation stops the running download or translation, waiting
|
||||||
|
// for it to wind down so that a new task can start right away.
|
||||||
func (a *App) CancelTranslation() {
|
func (a *App) CancelTranslation() {
|
||||||
a.tr.mu.Lock()
|
a.tr.mu.Lock()
|
||||||
defer a.tr.mu.Unlock()
|
cancel, done := a.tr.cancel, a.tr.done
|
||||||
if a.tr.cancel != nil {
|
a.tr.mu.Unlock()
|
||||||
a.tr.cancel()
|
if cancel == nil {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
}
|
}
|
||||||
|
|
||||||
// progressEmitter sends "translation-progress" events, at most every
|
// progressEmitter sends "translation-progress" events, at most every
|
||||||
@@ -136,8 +165,12 @@ func (a *App) progressEmitter() func(done, total int64) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DownloadTranslationModel downloads the model into the user's data
|
// DownloadTranslationModel downloads the model into the user's data
|
||||||
// directory. It returns false if the user cancelled.
|
// directory. An interrupted download resumes where it stopped. It returns
|
||||||
|
// false if the user cancelled.
|
||||||
func (a *App) DownloadTranslationModel() (bool, error) {
|
func (a *App) DownloadTranslationModel() (bool, error) {
|
||||||
|
if installedModel() != "" {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
ctx, err := a.startTask()
|
ctx, err := a.startTask()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -148,11 +181,30 @@ func (a *App) DownloadTranslationModel() (bool, error) {
|
|||||||
f.URL = modelURL + f.Name
|
f.URL = modelURL + f.Name
|
||||||
files[i] = f
|
files[i] = f
|
||||||
}
|
}
|
||||||
err = marian.Download(ctx, &http.Client{}, modelDirs()[0], files, a.progressEmitter())
|
// No total timeout (300 MB can take a while) but abort a stalled
|
||||||
if errors.Is(err, context.Canceled) {
|
// transfer: the watchdog is reset by every progress report.
|
||||||
return false, nil
|
ctx, stop := context.WithCancelCause(ctx)
|
||||||
|
defer stop(nil)
|
||||||
|
emit := a.progressEmitter()
|
||||||
|
stalled := time.AfterFunc(stallTimeout, func() { stop(errStalled) })
|
||||||
|
defer stalled.Stop()
|
||||||
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||||
|
transport.ResponseHeaderTimeout = stallTimeout
|
||||||
|
client := &http.Client{Transport: transport}
|
||||||
|
err = marian.Download(ctx, client, modelDirs()[0], files, func(done, total int64) {
|
||||||
|
stalled.Reset(stallTimeout)
|
||||||
|
emit(done, total)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(context.Cause(ctx), errStalled) {
|
||||||
|
return false, errStalled
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
}
|
}
|
||||||
return err == nil, err
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Translate translates the French text of the elements (flat indexes; all
|
// Translate translates the French text of the elements (flat indexes; all
|
||||||
@@ -164,8 +216,14 @@ func (a *App) Translate(doc screenplay.Document, elements []int, titlePage bool)
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer a.endTask()
|
defer a.endTask()
|
||||||
|
if installedModel() == "" {
|
||||||
|
return nil, errors.New("the translation model is not installed")
|
||||||
|
}
|
||||||
wails.EventsEmit(a.ctx, "translation-progress", 0, 0) // loading
|
wails.EventsEmit(a.ctx, "translation-progress", 0, 0) // loading
|
||||||
model, lexicon, err := a.translator()
|
model, lexicon, err := a.translator(ctx)
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -174,10 +232,8 @@ func (a *App) Translate(doc screenplay.Document, elements []int, titlePage bool)
|
|||||||
res, err := translate.Document(ctx, &doc, model, lexicon, translate.Options{
|
res, err := translate.Document(ctx, &doc, model, lexicon, translate.Options{
|
||||||
Elements: elements,
|
Elements: elements,
|
||||||
TitlePage: titlePage,
|
TitlePage: titlePage,
|
||||||
// The model splits its largest product across every CPU already:
|
Workers: translateWorkers(),
|
||||||
// half as many sentences at once keeps the machine responsive.
|
Progress: func(done, total int) { emit(int64(done), int64(total)) },
|
||||||
Workers: max(1, runtime.NumCPU()/2),
|
|
||||||
Progress: func(done, total int) { emit(int64(done), int64(total)) },
|
|
||||||
})
|
})
|
||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -188,39 +244,68 @@ func (a *App) Translate(doc screenplay.Document, elements []int, titlePage bool)
|
|||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// translateWorkers is how many sentences are translated at once: half the
|
||||||
|
// CPUs, and each one splits its decoding steps across its share of them.
|
||||||
|
func translateWorkers() int { return max(1, runtime.NumCPU()/2) }
|
||||||
|
|
||||||
// translator returns the model and the dictionaries that help tell French
|
// translator returns the model and the dictionaries that help tell French
|
||||||
// from English, loading them the first time.
|
// from English, loading them the first time. The load happens without
|
||||||
func (a *App) translator() (*marian.Model, translate.Lexicon, error) {
|
// a.tr.mu: reading 300 MB takes seconds, and CancelTranslation must stay
|
||||||
|
// responsive.
|
||||||
|
func (a *App) translator(ctx context.Context) (*marian.Model, translate.Lexicon, error) {
|
||||||
a.tr.mu.Lock()
|
a.tr.mu.Lock()
|
||||||
defer a.tr.mu.Unlock()
|
|
||||||
if a.tr.release != nil {
|
if a.tr.release != nil {
|
||||||
a.tr.release.Stop()
|
a.tr.release.Stop()
|
||||||
}
|
}
|
||||||
if a.tr.model == nil {
|
model, lexicon := a.tr.model, a.tr.lexicon
|
||||||
dir := installedModel()
|
a.tr.mu.Unlock()
|
||||||
if dir == "" {
|
if model != nil {
|
||||||
return nil, nil, errors.New("the translation model is not installed")
|
return model, lexicon, nil
|
||||||
}
|
|
||||||
m, err := marian.Load(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
a.tr.model = m
|
|
||||||
a.tr.lexicon = loadLexicon()
|
|
||||||
}
|
}
|
||||||
return a.tr.model, a.tr.lexicon, nil
|
dir := installedModel()
|
||||||
|
if dir == "" {
|
||||||
|
return nil, nil, errors.New("the translation model is not installed")
|
||||||
|
}
|
||||||
|
m, err := marian.Load(ctx, dir)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, nil, ctx.Err()
|
||||||
|
}
|
||||||
|
return nil, nil, fmt.Errorf("loading the translation model: %w", err)
|
||||||
|
}
|
||||||
|
// Every sentence worker gets an equal share of the CPUs for its
|
||||||
|
// decoding steps, so workers × threads never oversubscribe the machine.
|
||||||
|
m.SetThreads(max(1, runtime.NumCPU()/translateWorkers()))
|
||||||
|
lexicon = a.loadLexicon()
|
||||||
|
a.tr.mu.Lock()
|
||||||
|
if ctx.Err() == nil { // cancelled while loading: cache nothing
|
||||||
|
a.tr.model, a.tr.lexicon = m, lexicon
|
||||||
|
}
|
||||||
|
a.tr.mu.Unlock()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return m, lexicon, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// releaseLater frees the model once it has been unused for modelIdle.
|
||||||
func (a *App) releaseLater() {
|
func (a *App) releaseLater() {
|
||||||
a.tr.mu.Lock()
|
a.tr.mu.Lock()
|
||||||
defer a.tr.mu.Unlock()
|
defer a.tr.mu.Unlock()
|
||||||
a.tr.release = time.AfterFunc(modelIdle, func() {
|
a.tr.release = time.AfterFunc(modelIdle, a.releaseModel)
|
||||||
a.tr.mu.Lock()
|
}
|
||||||
defer a.tr.mu.Unlock()
|
|
||||||
if a.tr.cancel == nil {
|
// releaseModel frees the model when no task is running. When one is (a
|
||||||
a.tr.model, a.tr.lexicon = nil, nil
|
// download loads nothing but holds the task slot), it tries again after
|
||||||
}
|
// another modelIdle.
|
||||||
})
|
func (a *App) releaseModel() {
|
||||||
|
a.tr.mu.Lock()
|
||||||
|
defer a.tr.mu.Unlock()
|
||||||
|
if a.tr.cancel != nil {
|
||||||
|
a.tr.release = time.AfterFunc(modelIdle, a.releaseModel)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.tr.model, a.tr.lexicon = nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// lexicon tells French words from English ones with the installed
|
// lexicon tells French words from English ones with the installed
|
||||||
@@ -231,25 +316,32 @@ func (l lexicon) Knows(word string) (french, english bool) { return l.fr.Check(w
|
|||||||
|
|
||||||
// loadLexicon needs both a French and an English dictionary: with one
|
// loadLexicon needs both a French and an English dictionary: with one
|
||||||
// only, every word it knows would count for its language. Without them,
|
// only, every word it knows would count for its language. Without them,
|
||||||
// detection relies on function words and accents.
|
// detection relies on function words and accents. The dictionaries come
|
||||||
func loadLexicon() translate.Lexicon {
|
// from the same cache as the spell checker's: a Dictionary is read-only
|
||||||
|
// once loaded, so it can be shared.
|
||||||
|
func (a *App) loadLexicon() translate.Lexicon {
|
||||||
langs := spell.Available(dictionaryDirs())
|
langs := spell.Available(dictionaryDirs())
|
||||||
load := func(lang, preferred string) *spell.Dictionary {
|
pick := func(lang, preferred string) string {
|
||||||
i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred })
|
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return l.Code == preferred }); i >= 0 {
|
||||||
if i < 0 {
|
return langs[i].Code
|
||||||
i = slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") })
|
|
||||||
}
|
}
|
||||||
if i < 0 {
|
if i := slices.IndexFunc(langs, func(l spell.Language) bool { return strings.HasPrefix(l.Code, lang+"_") }); i >= 0 {
|
||||||
return nil
|
return langs[i].Code
|
||||||
}
|
}
|
||||||
d, err := spell.Load(langs[i].Aff, langs[i].Dic)
|
return ""
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return d
|
|
||||||
}
|
}
|
||||||
fr, en := load("fr", "fr_FR"), load("en", "en_US")
|
frCode, enCode := pick("fr", "fr_FR"), pick("en", "en_US")
|
||||||
if fr == nil || en == nil {
|
if frCode == "" || enCode == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
a.spellMu.Lock()
|
||||||
|
defer a.spellMu.Unlock()
|
||||||
|
fr, err := a.dictionaryLocked(frCode)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
en, err := a.dictionaryLocked(enCode)
|
||||||
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return lexicon{fr, en}
|
return lexicon{fr, en}
|
||||||
|
|||||||
Reference in new issue
Block a user