From 2838d58b1daa0f6a337125c5a64d06215901c5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 4 May 2019 18:25:56 +0200 Subject: i18n: Move the package below /langs To get fewer top level packages. --- langs/i18n/i18n.go | 117 +++++++++++++++++ langs/i18n/i18n_test.go | 262 ++++++++++++++++++++++++++++++++++++++ langs/i18n/translationProvider.go | 125 ++++++++++++++++++ 3 files changed, 504 insertions(+) create mode 100644 langs/i18n/i18n.go create mode 100644 langs/i18n/i18n_test.go create mode 100644 langs/i18n/translationProvider.go (limited to 'langs') diff --git a/langs/i18n/i18n.go b/langs/i18n/i18n.go new file mode 100644 index 000000000..5beef8683 --- /dev/null +++ b/langs/i18n/i18n.go @@ -0,0 +1,117 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package i18n + +import ( + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" + + "github.com/nicksnyder/go-i18n/i18n/bundle" + "github.com/nicksnyder/go-i18n/i18n/translation" +) + +var ( + i18nWarningLogger = helpers.NewDistinctFeedbackLogger() +) + +// Translator handles i18n translations. +type Translator struct { + translateFuncs map[string]bundle.TranslateFunc + cfg config.Provider + logger *loggers.Logger +} + +// NewTranslator creates a new Translator for the given language bundle and configuration. +func NewTranslator(b *bundle.Bundle, cfg config.Provider, logger *loggers.Logger) Translator { + t := Translator{cfg: cfg, logger: logger, translateFuncs: make(map[string]bundle.TranslateFunc)} + t.initFuncs(b) + return t +} + +// Func gets the translate func for the given language, or for the default +// configured language if not found. +func (t Translator) Func(lang string) bundle.TranslateFunc { + if f, ok := t.translateFuncs[lang]; ok { + return f + } + t.logger.INFO.Printf("Translation func for language %v not found, use default.", lang) + if f, ok := t.translateFuncs[t.cfg.GetString("defaultContentLanguage")]; ok { + return f + } + t.logger.INFO.Println("i18n not initialized; if you need string translations, check that you have a bundle in /i18n that matches the site language or the default language.") + return func(translationID string, args ...interface{}) string { + return "" + } + +} + +func (t Translator) initFuncs(bndl *bundle.Bundle) { + defaultContentLanguage := t.cfg.GetString("defaultContentLanguage") + + defaultT, err := bndl.Tfunc(defaultContentLanguage) + if err != nil { + t.logger.INFO.Printf("No translation bundle found for default language %q", defaultContentLanguage) + } + + translations := bndl.Translations() + + enableMissingTranslationPlaceholders := t.cfg.GetBool("enableMissingTranslationPlaceholders") + for _, lang := range bndl.LanguageTags() { + currentLang := lang + + t.translateFuncs[currentLang] = func(translationID string, args ...interface{}) string { + tFunc, err := bndl.Tfunc(currentLang) + if err != nil { + t.logger.WARN.Printf("could not load translations for language %q (%s), will use default content language.\n", lang, err) + } + + translated := tFunc(translationID, args...) + if translated != translationID { + return translated + } + // If there is no translation for translationID, + // then Tfunc returns translationID itself. + // But if user set same translationID and translation, we should check + // if it really untranslated: + if isIDTranslated(translations, currentLang, translationID) { + return translated + } + + if t.cfg.GetBool("logI18nWarnings") { + i18nWarningLogger.Printf("i18n|MISSING_TRANSLATION|%s|%s", currentLang, translationID) + } + if enableMissingTranslationPlaceholders { + return "[i18n] " + translationID + } + if defaultT != nil { + translated := defaultT(translationID, args...) + if translated != translationID { + return translated + } + if isIDTranslated(translations, defaultContentLanguage, translationID) { + return translated + } + } + return "" + } + } +} + +// If the translation map contains translationID for specified currentLang, +// then the translationID is actually translated. +func isIDTranslated(translations map[string]map[string]translation.Translation, lang, id string) bool { + _, contains := translations[lang][id] + return contains +} diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go new file mode 100644 index 000000000..b67cabc55 --- /dev/null +++ b/langs/i18n/i18n_test.go @@ -0,0 +1,262 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package i18n + +import ( + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/tpl/tplimpl" + + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/langs" + "github.com/spf13/afero" + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/deps" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs" + "github.com/stretchr/testify/require" +) + +var logger = loggers.NewErrorLogger() + +type i18nTest struct { + name string + data map[string][]byte + args interface{} + lang, id, expected, expectedFlag string +} + +var i18nTests = []i18nTest{ + // All translations present + { + name: "all-present", + data: map[string][]byte{ + "en.toml": []byte("[hello]\nother = \"Hello, World!\""), + "es.toml": []byte("[hello]\nother = \"¡Hola, Mundo!\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "¡Hola, Mundo!", + expectedFlag: "¡Hola, Mundo!", + }, + // Translation missing in current language but present in default + { + name: "present-in-default", + data: map[string][]byte{ + "en.toml": []byte("[hello]\nother = \"Hello, World!\""), + "es.toml": []byte("[goodbye]\nother = \"¡Adiós, Mundo!\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "Hello, World!", + expectedFlag: "[i18n] hello", + }, + // Translation missing in default language but present in current + { + name: "present-in-current", + data: map[string][]byte{ + "en.toml": []byte("[goodbye]\nother = \"Goodbye, World!\""), + "es.toml": []byte("[hello]\nother = \"¡Hola, Mundo!\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "¡Hola, Mundo!", + expectedFlag: "¡Hola, Mundo!", + }, + // Translation missing in both default and current language + { + name: "missing", + data: map[string][]byte{ + "en.toml": []byte("[goodbye]\nother = \"Goodbye, World!\""), + "es.toml": []byte("[goodbye]\nother = \"¡Adiós, Mundo!\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "", + expectedFlag: "[i18n] hello", + }, + // Default translation file missing or empty + { + name: "file-missing", + data: map[string][]byte{ + "en.toml": []byte(""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "", + expectedFlag: "[i18n] hello", + }, + // Context provided + { + name: "context-provided", + data: map[string][]byte{ + "en.toml": []byte("[wordCount]\nother = \"Hello, {{.WordCount}} people!\""), + "es.toml": []byte("[wordCount]\nother = \"¡Hola, {{.WordCount}} gente!\""), + }, + args: struct { + WordCount int + }{ + 50, + }, + lang: "es", + id: "wordCount", + expected: "¡Hola, 50 gente!", + expectedFlag: "¡Hola, 50 gente!", + }, + // Same id and translation in current language + // https://github.com/gohugoio/hugo/issues/2607 + { + name: "same-id-and-translation", + data: map[string][]byte{ + "es.toml": []byte("[hello]\nother = \"hello\""), + "en.toml": []byte("[hello]\nother = \"hi\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "hello", + expectedFlag: "hello", + }, + // Translation missing in current language, but same id and translation in default + { + name: "same-id-and-translation-default", + data: map[string][]byte{ + "es.toml": []byte("[bye]\nother = \"bye\""), + "en.toml": []byte("[hello]\nother = \"hello\""), + }, + args: nil, + lang: "es", + id: "hello", + expected: "hello", + expectedFlag: "[i18n] hello", + }, + // Unknown language code should get its plural spec from en + { + name: "unknown-language-code", + data: map[string][]byte{ + "en.toml": []byte(`[readingTime] +one ="one minute read" +other = "{{.Count}} minutes read"`), + "klingon.toml": []byte(`[readingTime] +one = "eitt minutt med lesing" +other = "{{ .Count }} minuttar lesing"`), + }, + args: 3, + lang: "klingon", + id: "readingTime", + expected: "3 minuttar lesing", + expectedFlag: "3 minuttar lesing", + }, +} + +func doTestI18nTranslate(t testing.TB, test i18nTest, cfg config.Provider) string { + tp := prepareTranslationProvider(t, test, cfg) + f := tp.t.Func(test.lang) + return f(test.id, test.args) + +} + +func prepareTranslationProvider(t testing.TB, test i18nTest, cfg config.Provider) *TranslationProvider { + assert := require.New(t) + fs := hugofs.NewMem(cfg) + + for file, content := range test.data { + err := afero.WriteFile(fs.Source, filepath.Join("i18n", file), []byte(content), 0755) + assert.NoError(err) + } + + tp := NewTranslationProvider() + depsCfg := newDepsConfig(tp, cfg, fs) + d, err := deps.New(depsCfg) + assert.NoError(err) + assert.NoError(d.LoadResources()) + + return tp +} + +func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) deps.DepsCfg { + l := langs.NewLanguage("en", cfg) + l.Set("i18nDir", "i18n") + return deps.DepsCfg{ + Language: l, + Site: htesting.NewTestHugoSite(), + Cfg: cfg, + Fs: fs, + Logger: logger, + TemplateProvider: tplimpl.DefaultTemplateProvider, + TranslationProvider: tp, + } +} + +func getConfig() *viper.Viper { + v := viper.New() + v.SetDefault("defaultContentLanguage", "en") + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + return v + +} + +func TestI18nTranslate(t *testing.T) { + var actual, expected string + v := getConfig() + + // Test without and with placeholders + for _, enablePlaceholders := range []bool{false, true} { + v.Set("enableMissingTranslationPlaceholders", enablePlaceholders) + + for _, test := range i18nTests { + if enablePlaceholders { + expected = test.expectedFlag + } else { + expected = test.expected + } + actual = doTestI18nTranslate(t, test, v) + require.Equal(t, expected, actual) + } + } +} + +func BenchmarkI18nTranslate(b *testing.B) { + v := getConfig() + for _, test := range i18nTests { + b.Run(test.name, func(b *testing.B) { + tp := prepareTranslationProvider(b, test, v) + b.ResetTimer() + for i := 0; i < b.N; i++ { + f := tp.t.Func(test.lang) + actual := f(test.id, test.args) + if actual != test.expected { + b.Fatalf("expected %v got %v", test.expected, actual) + } + } + }) + } + +} diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go new file mode 100644 index 000000000..74e144007 --- /dev/null +++ b/langs/i18n/translationProvider.go @@ -0,0 +1,125 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package i18n + +import ( + "errors" + + "github.com/gohugoio/hugo/common/herrors" + + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/source" + "github.com/nicksnyder/go-i18n/i18n/bundle" + "github.com/nicksnyder/go-i18n/i18n/language" + _errors "github.com/pkg/errors" +) + +// TranslationProvider provides translation handling, i.e. loading +// of bundles etc. +type TranslationProvider struct { + t Translator +} + +// NewTranslationProvider creates a new translation provider. +func NewTranslationProvider() *TranslationProvider { + return &TranslationProvider{} +} + +// Update updates the i18n func in the provided Deps. +func (tp *TranslationProvider) Update(d *deps.Deps) error { + sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs) + src := sp.NewFilesystem("") + + i18nBundle := bundle.New() + + en := language.GetPluralSpec("en") + if en == nil { + return errors.New("the English language has vanished like an old oak table") + } + var newLangs []string + + for _, r := range src.Files() { + currentSpec := language.GetPluralSpec(r.BaseFileName()) + if currentSpec == nil { + // This may is a language code not supported by go-i18n, it may be + // Klingon or ... not even a fake language. Make sure it works. + newLangs = append(newLangs, r.BaseFileName()) + } + } + + if len(newLangs) > 0 { + language.RegisterPluralSpec(newLangs, en) + } + + // The source files are ordered so the most important comes first. Since this is a + // last key win situation, we have to reverse the iteration order. + files := src.Files() + for i := len(files) - 1; i >= 0; i-- { + if err := addTranslationFile(i18nBundle, files[i]); err != nil { + return err + } + } + + tp.t = NewTranslator(i18nBundle, d.Cfg, d.Log) + + d.Translate = tp.t.Func(d.Language.Lang) + + return nil + +} + +func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error { + f, err := r.Open() + if err != nil { + return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName()) + } + err = bundle.ParseTranslationFileBytes(r.LogicalName(), helpers.ReaderToBytes(f)) + f.Close() + if err != nil { + return errWithFileContext(_errors.Wrapf(err, "failed to load translations"), r) + } + return nil +} + +// Clone sets the language func for the new language. +func (tp *TranslationProvider) Clone(d *deps.Deps) error { + d.Translate = tp.t.Func(d.Language.Lang) + + return nil +} + +func errWithFileContext(inerr error, r source.ReadableFile) error { + rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo) + if !ok { + return inerr + } + + realFilename := rfi.RealFilename() + f, err := r.Open() + if err != nil { + return inerr + } + defer f.Close() + + err, _ = herrors.WithFileContext( + inerr, + realFilename, + f, + herrors.SimpleLineMatcher) + + return err + +} -- cgit v1.2.3