i18n: Move the package below /langs

To get fewer top level packages.

3 files changed, 504 insertions, 0 deletions

diff --git a/langs/i18n/i18n.go b/langs/i18n/i18n.go
new file mode 100644
index 000000000..5beef8683
--- /dev/null
+++ b/langs/i18n/i18n.go
@@ -0,0 +1,117 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package i18n
+
+import (
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/nicksnyder/go-i18n/i18n/bundle"
+	"github.com/nicksnyder/go-i18n/i18n/translation"
+)
+
+var (
+	i18nWarningLogger = helpers.NewDistinctFeedbackLogger()
+)
+
+// Translator handles i18n translations.
+type Translator struct {
+	translateFuncs map[string]bundle.TranslateFunc
+	cfg            config.Provider
+	logger         *loggers.Logger
+}
+
+// NewTranslator creates a new Translator for the given language bundle and configuration.
+func NewTranslator(b *bundle.Bundle, cfg config.Provider, logger *loggers.Logger) Translator {
+	t := Translator{cfg: cfg, logger: logger, translateFuncs: make(map[string]bundle.TranslateFunc)}
+	t.initFuncs(b)
+	return t
+}
+
+// Func gets the translate func for the given language, or for the default
+// configured language if not found.
+func (t Translator) Func(lang string) bundle.TranslateFunc {
+	if f, ok := t.translateFuncs[lang]; ok {
+		return f
+	}
+	t.logger.INFO.Printf("Translation func for language %v not found, use default.", lang)
+	if f, ok := t.translateFuncs[t.cfg.GetString("defaultContentLanguage")]; ok {
+		return f
+	}
+	t.logger.INFO.Println("i18n not initialized; if you need string translations, check that you have a bundle in /i18n that matches the site language or the default language.")
+	return func(translationID string, args ...interface{}) string {
+		return ""
+	}
+
+}
+
+func (t Translator) initFuncs(bndl *bundle.Bundle) {
+	defaultContentLanguage := t.cfg.GetString("defaultContentLanguage")
+
+	defaultT, err := bndl.Tfunc(defaultContentLanguage)
+	if err != nil {
+		t.logger.INFO.Printf("No translation bundle found for default language %q", defaultContentLanguage)
+	}
+
+	translations := bndl.Translations()
+
+	enableMissingTranslationPlaceholders := t.cfg.GetBool("enableMissingTranslationPlaceholders")
+	for _, lang := range bndl.LanguageTags() {
+		currentLang := lang
+
+		t.translateFuncs[currentLang] = func(translationID string, args ...interface{}) string {
+			tFunc, err := bndl.Tfunc(currentLang)
+			if err != nil {
+				t.logger.WARN.Printf("could not load translations for language %q (%s), will use default content language.\n", lang, err)
+			}
+
+			translated := tFunc(translationID, args...)
+			if translated != translationID {
+				return translated
+			}
+			// If there is no translation for translationID,
+			// then Tfunc returns translationID itself.
+			// But if user set same translationID and translation, we should check
+			// if it really untranslated:
+			if isIDTranslated(translations, currentLang, translationID) {
+				return translated
+			}
+
+			if t.cfg.GetBool("logI18nWarnings") {
+				i18nWarningLogger.Printf("i18n|MISSING_TRANSLATION|%s|%s", currentLang, translationID)
+			}
+			if enableMissingTranslationPlaceholders {
+				return "[i18n] " + translationID
+			}
+			if defaultT != nil {
+				translated := defaultT(translationID, args...)
+				if translated != translationID {
+					return translated
+				}
+				if isIDTranslated(translations, defaultContentLanguage, translationID) {
+					return translated
+				}
+			}
+			return ""
+		}
+	}
+}
+
+// If the translation map contains translationID for specified currentLang,
+// then the translationID is actually translated.
+func isIDTranslated(translations map[string]map[string]translation.Translation, lang, id string) bool {
+	_, contains := translations[lang][id]
+	return contains
+}
diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go
new file mode 100644
index 000000000..b67cabc55
--- /dev/null
+++ b/langs/i18n/i18n_test.go
@@ -0,0 +1,262 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package i18n
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/gohugoio/hugo/tpl/tplimpl"
+
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/htesting"
+	"github.com/gohugoio/hugo/langs"
+	"github.com/spf13/afero"
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/deps"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/stretchr/testify/require"
+)
+
+var logger = loggers.NewErrorLogger()
+
+type i18nTest struct {
+	name                             string
+	data                             map[string][]byte
+	args                             interface{}
+	lang, id, expected, expectedFlag string
+}
+
+var i18nTests = []i18nTest{
+	// All translations present
+	{
+		name: "all-present",
+		data: map[string][]byte{
+			"en.toml": []byte("[hello]\nother = \"Hello, World!\""),
+			"es.toml": []byte("[hello]\nother = \"¡Hola, Mundo!\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "¡Hola, Mundo!",
+		expectedFlag: "¡Hola, Mundo!",
+	},
+	// Translation missing in current language but present in default
+	{
+		name: "present-in-default",
+		data: map[string][]byte{
+			"en.toml": []byte("[hello]\nother = \"Hello, World!\""),
+			"es.toml": []byte("[goodbye]\nother = \"¡Adiós, Mundo!\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "Hello, World!",
+		expectedFlag: "[i18n] hello",
+	},
+	// Translation missing in default language but present in current
+	{
+		name: "present-in-current",
+		data: map[string][]byte{
+			"en.toml": []byte("[goodbye]\nother = \"Goodbye, World!\""),
+			"es.toml": []byte("[hello]\nother = \"¡Hola, Mundo!\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "¡Hola, Mundo!",
+		expectedFlag: "¡Hola, Mundo!",
+	},
+	// Translation missing in both default and current language
+	{
+		name: "missing",
+		data: map[string][]byte{
+			"en.toml": []byte("[goodbye]\nother = \"Goodbye, World!\""),
+			"es.toml": []byte("[goodbye]\nother = \"¡Adiós, Mundo!\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "",
+		expectedFlag: "[i18n] hello",
+	},
+	// Default translation file missing or empty
+	{
+		name: "file-missing",
+		data: map[string][]byte{
+			"en.toml": []byte(""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "",
+		expectedFlag: "[i18n] hello",
+	},
+	// Context provided
+	{
+		name: "context-provided",
+		data: map[string][]byte{
+			"en.toml": []byte("[wordCount]\nother = \"Hello, {{.WordCount}} people!\""),
+			"es.toml": []byte("[wordCount]\nother = \"¡Hola, {{.WordCount}} gente!\""),
+		},
+		args: struct {
+			WordCount int
+		}{
+			50,
+		},
+		lang:         "es",
+		id:           "wordCount",
+		expected:     "¡Hola, 50 gente!",
+		expectedFlag: "¡Hola, 50 gente!",
+	},
+	// Same id and translation in current language
+	// https://github.com/gohugoio/hugo/issues/2607
+	{
+		name: "same-id-and-translation",
+		data: map[string][]byte{
+			"es.toml": []byte("[hello]\nother = \"hello\""),
+			"en.toml": []byte("[hello]\nother = \"hi\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "hello",
+		expectedFlag: "hello",
+	},
+	// Translation missing in current language, but same id and translation in default
+	{
+		name: "same-id-and-translation-default",
+		data: map[string][]byte{
+			"es.toml": []byte("[bye]\nother = \"bye\""),
+			"en.toml": []byte("[hello]\nother = \"hello\""),
+		},
+		args:         nil,
+		lang:         "es",
+		id:           "hello",
+		expected:     "hello",
+		expectedFlag: "[i18n] hello",
+	},
+	// Unknown language code should get its plural spec from en
+	{
+		name: "unknown-language-code",
+		data: map[string][]byte{
+			"en.toml": []byte(`[readingTime]
+one ="one minute read"
+other = "{{.Count}} minutes read"`),
+			"klingon.toml": []byte(`[readingTime]
+one =  "eitt minutt med lesing"
+other = "{{ .Count }} minuttar lesing"`),
+		},
+		args:         3,
+		lang:         "klingon",
+		id:           "readingTime",
+		expected:     "3 minuttar lesing",
+		expectedFlag: "3 minuttar lesing",
+	},
+}
+
+func doTestI18nTranslate(t testing.TB, test i18nTest, cfg config.Provider) string {
+	tp := prepareTranslationProvider(t, test, cfg)
+	f := tp.t.Func(test.lang)
+	return f(test.id, test.args)
+
+}
+
+func prepareTranslationProvider(t testing.TB, test i18nTest, cfg config.Provider) *TranslationProvider {
+	assert := require.New(t)
+	fs := hugofs.NewMem(cfg)
+
+	for file, content := range test.data {
+		err := afero.WriteFile(fs.Source, filepath.Join("i18n", file), []byte(content), 0755)
+		assert.NoError(err)
+	}
+
+	tp := NewTranslationProvider()
+	depsCfg := newDepsConfig(tp, cfg, fs)
+	d, err := deps.New(depsCfg)
+	assert.NoError(err)
+	assert.NoError(d.LoadResources())
+
+	return tp
+}
+
+func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) deps.DepsCfg {
+	l := langs.NewLanguage("en", cfg)
+	l.Set("i18nDir", "i18n")
+	return deps.DepsCfg{
+		Language:            l,
+		Site:                htesting.NewTestHugoSite(),
+		Cfg:                 cfg,
+		Fs:                  fs,
+		Logger:              logger,
+		TemplateProvider:    tplimpl.DefaultTemplateProvider,
+		TranslationProvider: tp,
+	}
+}
+
+func getConfig() *viper.Viper {
+	v := viper.New()
+	v.SetDefault("defaultContentLanguage", "en")
+	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+	v.Set("assetDir", "assets")
+	v.Set("resourceDir", "resources")
+	v.Set("publishDir", "public")
+	return v
+
+}
+
+func TestI18nTranslate(t *testing.T) {
+	var actual, expected string
+	v := getConfig()
+
+	// Test without and with placeholders
+	for _, enablePlaceholders := range []bool{false, true} {
+		v.Set("enableMissingTranslationPlaceholders", enablePlaceholders)
+
+		for _, test := range i18nTests {
+			if enablePlaceholders {
+				expected = test.expectedFlag
+			} else {
+				expected = test.expected
+			}
+			actual = doTestI18nTranslate(t, test, v)
+			require.Equal(t, expected, actual)
+		}
+	}
+}
+
+func BenchmarkI18nTranslate(b *testing.B) {
+	v := getConfig()
+	for _, test := range i18nTests {
+		b.Run(test.name, func(b *testing.B) {
+			tp := prepareTranslationProvider(b, test, v)
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				f := tp.t.Func(test.lang)
+				actual := f(test.id, test.args)
+				if actual != test.expected {
+					b.Fatalf("expected %v got %v", test.expected, actual)
+				}
+			}
+		})
+	}
+
+}
diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go
new file mode 100644
index 000000000..74e144007
--- /dev/null
+++ b/langs/i18n/translationProvider.go
@@ -0,0 +1,125 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package i18n
+
+import (
+	"errors"
+
+	"github.com/gohugoio/hugo/common/herrors"
+
+	"github.com/gohugoio/hugo/deps"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/source"
+	"github.com/nicksnyder/go-i18n/i18n/bundle"
+	"github.com/nicksnyder/go-i18n/i18n/language"
+	_errors "github.com/pkg/errors"
+)
+
+// TranslationProvider provides translation handling, i.e. loading
+// of bundles etc.
+type TranslationProvider struct {
+	t Translator
+}
+
+// NewTranslationProvider creates a new translation provider.
+func NewTranslationProvider() *TranslationProvider {
+	return &TranslationProvider{}
+}
+
+// Update updates the i18n func in the provided Deps.
+func (tp *TranslationProvider) Update(d *deps.Deps) error {
+	sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs)
+	src := sp.NewFilesystem("")
+
+	i18nBundle := bundle.New()
+
+	en := language.GetPluralSpec("en")
+	if en == nil {
+		return errors.New("the English language has vanished like an old oak table")
+	}
+	var newLangs []string
+
+	for _, r := range src.Files() {
+		currentSpec := language.GetPluralSpec(r.BaseFileName())
+		if currentSpec == nil {
+			// This may is a language code not supported by go-i18n, it may be
+			// Klingon or ... not even a fake language. Make sure it works.
+			newLangs = append(newLangs, r.BaseFileName())
+		}
+	}
+
+	if len(newLangs) > 0 {
+		language.RegisterPluralSpec(newLangs, en)
+	}
+
+	// The source files are ordered so the most important comes first. Since this is a
+	// last key win situation, we have to reverse the iteration order.
+	files := src.Files()
+	for i := len(files) - 1; i >= 0; i-- {
+		if err := addTranslationFile(i18nBundle, files[i]); err != nil {
+			return err
+		}
+	}
+
+	tp.t = NewTranslator(i18nBundle, d.Cfg, d.Log)
+
+	d.Translate = tp.t.Func(d.Language.Lang)
+
+	return nil
+
+}
+
+func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error {
+	f, err := r.Open()
+	if err != nil {
+		return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName())
+	}
+	err = bundle.ParseTranslationFileBytes(r.LogicalName(), helpers.ReaderToBytes(f))
+	f.Close()
+	if err != nil {
+		return errWithFileContext(_errors.Wrapf(err, "failed to load translations"), r)
+	}
+	return nil
+}
+
+// Clone sets the language func for the new language.
+func (tp *TranslationProvider) Clone(d *deps.Deps) error {
+	d.Translate = tp.t.Func(d.Language.Lang)
+
+	return nil
+}
+
+func errWithFileContext(inerr error, r source.ReadableFile) error {
+	rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo)
+	if !ok {
+		return inerr
+	}
+
+	realFilename := rfi.RealFilename()
+	f, err := r.Open()
+	if err != nil {
+		return inerr
+	}
+	defer f.Close()
+
+	err, _ = herrors.WithFileContext(
+		inerr,
+		realFilename,
+		f,
+		herrors.SimpleLineMatcher)
+
+	return err
+
+}