js.Build: Generate tsconfig files

Updates #7777

Added support to allow SourceMap files to be external to the build.
In addition added more information when the js compilation has an error.
Correctly append sourceMappingURL to output file.
Fix merge conflict.

2 files changed, 373 insertions, 9 deletions

diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go
index e4b4b1c20..d316bc85b 100644
--- a/resources/resource_transformers/js/build.go
+++ b/resources/resource_transformers/js/build.go
@@ -14,11 +14,16 @@
 package js
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
+	"path/filepath"
+	"reflect"
 	"strings"
 
+	"github.com/achiku/varfmt"
 	"github.com/spf13/cast"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -33,6 +38,7 @@ import (
 	"github.com/gohugoio/hugo/resources/resource"
 )
 
+// Options esbuild configuration
 type Options struct {
 	// If not set, the source path will be used as the base target path.
 	// Note that the target path's extension may change if the target MIME type
@@ -42,7 +48,7 @@ type Options struct {
 	// Whether to minify to output.
 	Minify bool
 
-	// Whether to write mapfiles (currently inline only)
+	// Whether to write mapfiles
 	SourceMap string
 
 	// The language target.
@@ -61,6 +67,9 @@ type Options struct {
 	// User defined symbols.
 	Defines map[string]interface{}
 
+	// User defined data (must be JSON marshall'able)
+	Data interface{}
+
 	// What to use instead of React.createElement.
 	JSXFactory string
 
@@ -72,6 +81,8 @@ type Options struct {
 	contents   string
 	sourcefile string
 	resolveDir string
+	workDir    string
+	tsConfig   string
 }
 
 func decodeOptions(m map[string]interface{}) (Options, error) {
@@ -91,11 +102,13 @@ func decodeOptions(m map[string]interface{}) (Options, error) {
 	return opts, nil
 }
 
+// Client context for esbuild
 type Client struct {
 	rs  *resources.Spec
 	sfs *filesystems.SourceFilesystem
 }
 
+// New create new client context
 func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client {
 	return &Client{rs: rs, sfs: fs}
 }
@@ -110,6 +123,13 @@ func (t *buildTransformation) Key() internal.ResourceTransformationKey {
 	return internal.NewResourceTransformationKey("jsbuild", t.optsm)
 }
 
+func appendExts(list []string, rel string) []string {
+	for _, ext := range []string{".tsx", ".ts", ".jsx", ".mjs", ".cjs", ".js", ".json"} {
+		list = append(list, fmt.Sprintf("%s/index%s", rel, ext))
+	}
+	return list
+}
+
 func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error {
 	ctx.OutMediaType = media.JavascriptType
 
@@ -129,25 +149,345 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx
 		return err
 	}
 
-	sdir, sfile := path.Split(ctx.SourcePath)
+	sdir, sfile := filepath.Split(t.sfs.RealFilename(ctx.SourcePath))
+	opts.workDir, err = filepath.Abs(t.rs.WorkingDir)
+	if err != nil {
+		return err
+	}
+
 	opts.sourcefile = sfile
-	opts.resolveDir = t.sfs.RealFilename(sdir)
+	opts.resolveDir = sdir
 	opts.contents = string(src)
 	opts.mediaType = ctx.InMediaType
 
+	// Create new temporary tsconfig file
+	newTSConfig, err := ioutil.TempFile("", "tsconfig.*.json")
+	if err != nil {
+		return err
+	}
+
+	filesToDelete := make([]*os.File, 0)
+
+	defer func() {
+		for _, file := range filesToDelete {
+			os.Remove(file.Name())
+		}
+	}()
+
+	filesToDelete = append(filesToDelete, newTSConfig)
+	configDir, _ := filepath.Split(newTSConfig.Name())
+
+	// Search for the innerMost tsconfig or jsconfig
+	innerTsConfig := ""
+	tsDir := opts.resolveDir
+	baseURLAbs := configDir
+	baseURL := "."
+	for tsDir != "." {
+		tryTsConfig := path.Join(tsDir, "tsconfig.json")
+		_, err := os.Stat(tryTsConfig)
+		if err != nil {
+			tryTsConfig := path.Join(tsDir, "jsconfig.json")
+			_, err = os.Stat(tryTsConfig)
+			if err == nil {
+				innerTsConfig = tryTsConfig
+				baseURLAbs = tsDir
+				break
+			}
+		} else {
+			innerTsConfig = tryTsConfig
+			baseURLAbs = tsDir
+			break
+		}
+		if tsDir == opts.workDir {
+			break
+		}
+		tsDir = path.Dir(tsDir)
+	}
+
+	// Resolve paths for @assets and @js (@js is just an alias for assets/js)
+	dirs := make([]string, 0)
+	rootPaths := make([]string, 0)
+	for _, dir := range t.sfs.RealDirs(".") {
+		rootDir := dir
+		if !strings.HasSuffix(dir, "package.json") {
+			dirs = append(dirs, dir)
+		} else {
+			rootDir, _ = path.Split(dir)
+		}
+		nodeModules := path.Join(rootDir, "node_modules")
+		if _, err := os.Stat(nodeModules); err == nil {
+			rootPaths = append(rootPaths, nodeModules)
+		}
+	}
+
+	// Construct new temporary tsconfig file content
+	config := make(map[string]interface{})
+	if innerTsConfig != "" {
+		oldConfig, err := ioutil.ReadFile(innerTsConfig)
+		if err == nil {
+			// If there is an error, it just means there is no config file here.
+			// Since we're also using the tsConfig file path to detect where
+			// to put the temp file, this is ok.
+			err = json.Unmarshal(oldConfig, &config)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if config["compilerOptions"] == nil {
+		config["compilerOptions"] = map[string]interface{}{}
+	}
+
+	// Assign new global paths to the config file while reading existing ones.
+	compilerOptions := config["compilerOptions"].(map[string]interface{})
+
+	// Handle original baseUrl if it's there
+	if compilerOptions["baseUrl"] != nil {
+		baseURL = compilerOptions["baseUrl"].(string)
+		oldBaseURLAbs := path.Join(tsDir, baseURL)
+		rel, _ := filepath.Rel(configDir, oldBaseURLAbs)
+		configDir = oldBaseURLAbs
+		baseURLAbs = configDir
+		if "/" != helpers.FilePathSeparator {
+			// On windows we need to use slashes instead of backslash
+			rel = strings.ReplaceAll(rel, helpers.FilePathSeparator, "/")
+		}
+		if rel != "" {
+			if strings.HasPrefix(rel, ".") {
+				baseURL = rel
+			} else {
+				baseURL = fmt.Sprintf("./%s", rel)
+			}
+		}
+		compilerOptions["baseUrl"] = baseURL
+	} else {
+		compilerOptions["baseUrl"] = baseURL
+	}
+
+	jsRel := func(refPath string) string {
+		rel, _ := filepath.Rel(configDir, refPath)
+		if "/" != helpers.FilePathSeparator {
+			// On windows we need to use slashes instead of backslash
+			rel = strings.ReplaceAll(rel, helpers.FilePathSeparator, "/")
+		}
+		if rel != "" {
+			if !strings.HasPrefix(rel, ".") {
+				rel = fmt.Sprintf("./%s", rel)
+			}
+		} else {
+			rel = "."
+		}
+		return rel
+	}
+
+	// Handle possible extends
+	if config["extends"] != nil {
+		extends := config["extends"].(string)
+		extendsAbs := path.Join(tsDir, extends)
+		rel := jsRel(extendsAbs)
+		config["extends"] = rel
+	}
+
+	var optionsPaths map[string]interface{}
+	// Get original paths if they exist
+	if compilerOptions["paths"] != nil {
+		optionsPaths = compilerOptions["paths"].(map[string]interface{})
+	} else {
+		optionsPaths = make(map[string]interface{})
+	}
+	compilerOptions["paths"] = optionsPaths
+
+	assets := make([]string, 0)
+	assetsExact := make([]string, 0)
+	js := make([]string, 0)
+	jsExact := make([]string, 0)
+	for _, dir := range dirs {
+		rel := jsRel(dir)
+		assets = append(assets, fmt.Sprintf("%s/*", rel))
+		assetsExact = appendExts(assetsExact, rel)
+
+		rel = jsRel(filepath.Join(dir, "js"))
+		js = append(js, fmt.Sprintf("%s/*", rel))
+		jsExact = appendExts(jsExact, rel)
+	}
+
+	optionsPaths["@assets/*"] = assets
+	optionsPaths["@js/*"] = js
+
+	// Make @js and @assets absolue matches search for index files
+	// to get around the problem in ESBuild resolving folders as index files.
+	optionsPaths["@assets"] = assetsExact
+	optionsPaths["@js"] = jsExact
+
+	var newDataFile *os.File
+	if opts.Data != nil {
+		// Create a data file
+		lines := make([]string, 0)
+		lines = append(lines, "// auto generated data import")
+		exports := make([]string, 0)
+		keys := make(map[string]bool)
+
+		var bytes []byte
+
+		conv := reflect.ValueOf(opts.Data)
+		convType := conv.Kind()
+		if convType == reflect.Interface {
+			if conv.IsNil() {
+				conv = reflect.Value{}
+			}
+		}
+
+		if conv.Kind() != reflect.Map {
+			// Write out as single JSON file
+			newDataFile, err = ioutil.TempFile("", "data.*.json")
+			// Output the data
+			bytes, err = json.MarshalIndent(conv.InterfaceData(), "", "  ")
+			if err != nil {
+				return err
+			}
+		} else {
+			// Try to allow tree shaking at the root
+			newDataFile, err = ioutil.TempFile(configDir, "data.*.js")
+			for _, key := range conv.MapKeys() {
+				strKey := key.Interface().(string)
+				if keys[strKey] {
+					continue
+				}
+				keys[strKey] = true
+
+				value := conv.MapIndex(key)
+
+				keyVar := varfmt.PublicVarName(strKey)
+
+				// Output the data
+				bytes, err := json.MarshalIndent(value.Interface(), "", "  ")
+				if err != nil {
+					return err
+				}
+				jsonValue := string(bytes)
+
+				lines = append(lines, fmt.Sprintf("export const %s = %s;", keyVar, jsonValue))
+				exports = append(exports, fmt.Sprintf("  %s,", keyVar))
+				if strKey != keyVar {
+					exports = append(exports, fmt.Sprintf("  [\"%s\"]: %s,", strKey, keyVar))
+				}
+			}
+
+			lines = append(lines, "const all = {")
+			for _, line := range exports {
+				lines = append(lines, line)
+			}
+			lines = append(lines, "};")
+			lines = append(lines, "export default all;")
+
+			bytes = []byte(strings.Join(lines, "\n"))
+		}
+
+		// Write tsconfig file
+		_, err = newDataFile.Write(bytes)
+		if err != nil {
+			return err
+		}
+		err = newDataFile.Close()
+		if err != nil {
+			return err
+		}
+
+		// Link this file into `import data from "@data"`
+		dataFiles := make([]string, 1)
+		rel, _ := filepath.Rel(baseURLAbs, newDataFile.Name())
+		dataFiles[0] = rel
+		optionsPaths["@data"] = dataFiles
+
+		filesToDelete = append(filesToDelete, newDataFile)
+	}
+
+	if len(rootPaths) > 0 {
+		// This will allow import "react" to resolve a react module that's
+		// either in the root node_modules or in one of the hugo mods.
+		optionsPaths["*"] = rootPaths
+	}
+
+	// Output the new config file
+	bytes, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	// Write tsconfig file
+	_, err = newTSConfig.Write(bytes)
+	if err != nil {
+		return err
+	}
+	err = newTSConfig.Close()
+	if err != nil {
+		return err
+	}
+
+	// Tell ESBuild about this new config file to use
+	opts.tsConfig = newTSConfig.Name()
+
 	buildOptions, err := toBuildOptions(opts)
 	if err != nil {
+		os.Remove(opts.tsConfig)
 		return err
 	}
 
 	result := api.Build(buildOptions)
+
+	if len(result.Warnings) > 0 {
+		for _, value := range result.Warnings {
+			if value.Location != nil {
+				t.rs.Logger.WARN.Println(fmt.Sprintf("%s:%d: WARN: %s",
+					filepath.Join(sdir, value.Location.File),
+					value.Location.Line, value.Text))
+				t.rs.Logger.WARN.Println("  ", value.Location.LineText)
+			} else {
+				t.rs.Logger.WARN.Println(fmt.Sprintf("%s: WARN: %s",
+					sdir,
+					value.Text))
+			}
+		}
+	}
 	if len(result.Errors) > 0 {
-		return fmt.Errorf("%s", result.Errors[0].Text)
+		output := result.Errors[0].Text
+		for _, value := range result.Errors {
+			var line string
+			if value.Location != nil {
+				line = fmt.Sprintf("%s:%d ERROR: %s",
+					filepath.Join(sdir, value.Location.File),
+					value.Location.Line, value.Text)
+			} else {
+				line = fmt.Sprintf("%s ERROR: %s",
+					sdir,
+					value.Text)
+			}
+			t.rs.Logger.ERROR.Println(line)
+			output = fmt.Sprintf("%s\n%s", output, line)
+			if value.Location != nil {
+				t.rs.Logger.ERROR.Println("  ", value.Location.LineText)
+			}
+		}
+		return fmt.Errorf("%s", output)
+	}
+
+	if buildOptions.Outfile != "" {
+		_, tfile := path.Split(opts.TargetPath)
+		output := fmt.Sprintf("%s//# sourceMappingURL=%s\n",
+			string(result.OutputFiles[1].Contents), tfile+".map")
+		_, err := ctx.To.Write([]byte(output))
+		if err != nil {
+			return err
+		}
+		ctx.PublishSourceMap(string(result.OutputFiles[0].Contents))
+	} else {
+		ctx.To.Write(result.OutputFiles[0].Contents)
 	}
-	ctx.To.Write(result.OutputFiles[0].Contents)
 	return nil
 }
 
+// Process process esbuild transform
 func (c *Client) Process(res resources.ResourceTransformer, opts map[string]interface{}) (resource.Resource, error) {
 	return res.Transform(
 		&buildTransformation{rs: c.rs, sfs: c.sfs, optsm: opts},
@@ -212,7 +552,6 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) {
 	default:
 		err = fmt.Errorf("unsupported script output format: %q", opts.Format)
 		return
-
 	}
 
 	var defines map[string]string
@@ -220,10 +559,19 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) {
 		defines = cast.ToStringMapString(opts.Defines)
 	}
 
+	// By default we only need to specify outDir and no outFile
+	var outDir = opts.outDir
+	var outFile = ""
 	var sourceMap api.SourceMap
 	switch opts.SourceMap {
 	case "inline":
 		sourceMap = api.SourceMapInline
+	case "external":
+		// When doing external sourcemaps we should specify
+		// out file and no out dir
+		sourceMap = api.SourceMapExternal
+		outFile = filepath.Join(opts.workDir, opts.TargetPath)
+		outDir = ""
 	case "":
 		sourceMap = api.SourceMapNone
 	default:
@@ -232,7 +580,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) {
 	}
 
 	buildOptions = api.BuildOptions{
-		Outfile: "",
+		Outfile: outFile,
 		Bundle:  true,
 
 		Target:    target,
@@ -243,7 +591,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) {
 		MinifyIdentifiers: opts.Minify,
 		MinifySyntax:      opts.Minify,
 
-		Outdir:  opts.outDir,
+		Outdir:  outDir,
 		Defines: defines,
 
 		Externals: opts.Externals,
@@ -251,7 +599,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) {
 		JSXFactory:  opts.JSXFactory,
 		JSXFragment: opts.JSXFragment,
 
-		//Tsconfig: opts.TSConfig,
+		Tsconfig: opts.tsConfig,
 
 		Stdin: &api.StdinOptions{
 			Contents:   opts.contents,
diff --git a/resources/resource_transformers/js/build_test.go b/resources/resource_transformers/js/build_test.go
index c04c0ed12..8839c646e 100644
--- a/resources/resource_transformers/js/build_test.go
+++ b/resources/resource_transformers/js/build_test.go
@@ -77,4 +77,20 @@ func TestToBuildOptions(t *testing.T) {
 		Sourcemap:         api.SourceMapInline,
 		Stdin:             &api.StdinOptions{},
 	})
+
+	opts, err = toBuildOptions(Options{
+		Target: "es2018", Format: "cjs", Minify: true, mediaType: media.JavascriptType,
+		SourceMap: "external"})
+	c.Assert(err, qt.IsNil)
+	c.Assert(opts, qt.DeepEquals, api.BuildOptions{
+		Bundle:            true,
+		Target:            api.ES2018,
+		Format:            api.FormatCommonJS,
+		MinifyIdentifiers: true,
+		MinifySyntax:      true,
+		MinifyWhitespace:  true,
+		Sourcemap:         api.SourceMapExternal,
+		Stdin:             &api.StdinOptions{},
+	})
+
 }