[Vue] Chunk UMD JavaScript into a set of asynchronously loaded files (#18761)

* proof of concept for chunking UMD JavaScript into a set of files that are loaded asynchronously

* take into account alternative plugin directories

* make chunk count and load umds individually configurable (undocumented config) and get to work

* fix a bug and add chunk JS sizes to output of development:compute-js-asset-size

* document

* fill out TODO documentation

* make sure cache buster is added to chunk script srcs

* add some checks in case a chunk does not exist on disk (happens during some tests)

* use realpath on test PIWIK_INCLUDE_PATH so search/replace on paths for relative path will work

* add integration test and get to pass

* fix for when disable_merged_assets=1

* fix condition

* fix ui test failure

* update screenshot

Co-authored-by: sgiehl <stefan@matomo.org>

5 files changed, 386 insertions, 66 deletions

diff --git a/core/AssetManager.php b/core/AssetManager.php
index 3e174718e2..c2ad38ef96 100644
--- a/core/AssetManager.php
+++ b/core/AssetManager.php
@@ -16,6 +16,7 @@ use Piwik\AssetManager\UIAssetCacheBuster;
 use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher;
 use Piwik\AssetManager\UIAssetFetcher\StaticUIAssetFetcher;
 use Piwik\AssetManager\UIAssetFetcher\StylesheetUIAssetFetcher;
+use Piwik\AssetManager\UIAssetFetcher\PluginUmdAssetFetcher;
 use Piwik\AssetManager\UIAssetFetcher;
 use Piwik\AssetManager\UIAssetMerger\JScriptUIAssetMerger;
 use Piwik\AssetManager\UIAssetMerger\StylesheetUIAssetMerger;
@@ -44,9 +45,11 @@ class AssetManager extends Singleton
 
     const CSS_IMPORT_DIRECTIVE = "<link rel=\"stylesheet\" type=\"text/css\" href=\"%s\" />\n";
     const JS_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\"></script>\n";
+    const JS_DEFER_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\" defer></script>\n";
     const GET_CSS_MODULE_ACTION = "index.php?module=Proxy&action=getCss";
     const GET_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getCoreJs";
     const GET_NON_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getNonCoreJs";
+    const GET_JS_UMD_MODULE_ACTION = "index.php?module=Proxy&action=getUmdJs&chunk=";
 
     /**
      * @var UIAssetCacheBuster
@@ -149,11 +152,27 @@ class AssetManager extends Singleton
         } else {
             $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_CORE_JS_MODULE_ACTION);
             $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_NON_CORE_JS_MODULE_ACTION);
+
+            $result .= $this->getPluginUmdChunks();
         }
 
         return $result;
     }
 
+    private function getPluginUmdChunks()
+    {
+        $fetcher = $this->getPluginUmdJScriptFetcher();
+
+        $chunks = $fetcher->getChunkFiles();
+
+        $result = '';
+        foreach ($chunks as $chunk) {
+            $src = self::GET_JS_UMD_MODULE_ACTION . urlencode($chunk->getChunkName());
+            $result .= sprintf(self::JS_DEFER_IMPORT_DIRECTIVE, $src);
+        }
+        return $result;
+    }
+
     /**
      * Return the base.less compiled to css
      *
@@ -212,7 +231,22 @@ class AssetManager extends Singleton
     }
 
     /**
-     * @param boolean $core
+     * Return a chunk JS merged file absolute location.
+     * If there is none, the generation process will be triggered.
+     *
+     * @param string $chunk The name of the chunk. Will either be a plugin name or an integer.
+     * @return UIAsset
+     */
+    public function getMergedJavaScriptChunk($chunk)
+    {
+        $assetFetcher = $this->getPluginUmdJScriptFetcher($chunk);
+        $outputFile = $assetFetcher->getRequestedChunkOutputFile();
+
+        return $this->getMergedJavascript($assetFetcher, $this->getMergedUIAsset($outputFile));
+    }
+
+    /**
+     * @param boolean|"all" $core
      * @return string[]
      */
     public function getLoadedPlugins($core)
@@ -223,7 +257,7 @@ class AssetManager extends Singleton
             $pluginName = $plugin->getPluginName();
             $pluginIsCore = Manager::getInstance()->isPluginBundledWithCore($pluginName);
 
-            if (($pluginIsCore && $core) || (!$pluginIsCore && !$core)) {
+            if ($core === 'all' || ($pluginIsCore && $core) || (!$pluginIsCore && !$core)) {
                 $loadedPlugins[] = $pluginName;
             }
         }
@@ -316,7 +350,8 @@ class AssetManager extends Singleton
     {
         return
             $this->getIndividualJsIncludesFromAssetFetcher($this->getCoreJScriptFetcher()) .
-            $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher());
+            $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher()) .
+            $this->getIndividualJsIncludesFromAssetFetcher($this->getPluginUmdJScriptFetcher());
     }
 
     /**
@@ -347,6 +382,11 @@ class AssetManager extends Singleton
         return new JScriptUIAssetFetcher($this->getLoadedPlugins(false), $this->theme);
     }
 
+    protected function getPluginUmdJScriptFetcher($chunk = null)
+    {
+        return new PluginUmdAssetFetcher($this->getLoadedPlugins('all'), $this->theme, $chunk);
+    }
+
     /**
      * @param string $pluginName
      * @return boolean
diff --git a/core/AssetManager/UIAssetFetcher/Chunk.php b/core/AssetManager/UIAssetFetcher/Chunk.php
new file mode 100644
index 0000000000..db42428baa
--- /dev/null
+++ b/core/AssetManager/UIAssetFetcher/Chunk.php
@@ -0,0 +1,69 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\AssetManager\UIAssetFetcher;
+
+class Chunk
+{
+    /**
+     * @var string
+     */
+    private $chunkName;
+
+    /**
+     * @var string[]
+     */
+    private $files;
+
+    public function __construct($chunkName, $files)
+    {
+        $this->chunkName = $chunkName;
+        $this->files = $files;
+    }
+
+    /**
+     * @return string
+     */
+    public function getOutputFile(): string
+    {
+        return "asset_manager_chunk.{$this->chunkName}.js";
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getFiles(): array
+    {
+        return $this->files;
+    }
+
+    /**
+     * @param string[] $files
+     */
+    public function setFiles(array $files): void
+    {
+        $this->files = $files;
+    }
+
+    /**
+     * @return string
+     */
+    public function getChunkName(): string
+    {
+        return $this->chunkName;
+    }
+
+    /**
+     * @param string $chunkName
+     */
+    public function setChunkName(string $chunkName): void
+    {
+        $this->chunkName = $chunkName;
+    }
+}
\ No newline at end of file
diff --git a/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php b/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php
index b9a4b3838b..5a31a25505 100644
--- a/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php
+++ b/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php
@@ -18,7 +18,6 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
     protected function retrieveFileLocations()
     {
         if (!empty($this->plugins)) {
-
             /**
              * Triggered when gathering the list of all JavaScript files needed by Piwik
              * and its plugins.
@@ -44,8 +43,6 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
              * @param string[] $jsFiles The JavaScript files to load.
              */
              Piwik::postEvent('AssetManager.getJavaScriptFiles', array(&$this->fileLocations), null, $this->plugins);
-
-             $this->addUmdFilesIfDetected($this->plugins);
         }
 
         $this->addThemeFiles();
@@ -96,64 +93,4 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
             'tests/',
         );
     }
-
-    private function addUmdFilesIfDetected($plugins)
-    {
-        $plugins = self::orderPluginsByPluginDependencies($plugins);
-
-        foreach ($plugins as $plugin) {
-            $devUmd = "plugins/$plugin/vue/dist/$plugin.development.umd.js";
-            $minifiedUmd = "plugins/$plugin/vue/dist/$plugin.umd.min.js";
-            $umdSrcFolder = "plugins/$plugin/vue/src";
-
-            // in case there are dist files but no src files, which can happen during development
-            if (is_dir($umdSrcFolder)) {
-                if (Development::isEnabled() && is_file(PIWIK_INCLUDE_PATH . '/' . $devUmd)) {
-                    $this->fileLocations[$plugin] = $devUmd;
-                } else if (is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) {
-                    $this->fileLocations[$plugin] = $minifiedUmd;
-                }
-            }
-        }
-    }
-
-    public static function orderPluginsByPluginDependencies($plugins)
-    {
-        $result = [];
-
-        while (!empty($plugins)) {
-            self::visitPlugin(reset($plugins), $plugins, $result);
-        }
-
-        return $result;
-    }
-
-    private static function visitPlugin($plugin, &$plugins, &$result)
-    {
-        // remove the plugin from the array of plugins to visit
-        $index = array_search($plugin, $plugins);
-        if ($index !== false) {
-            unset($plugins[$index]);
-        } else {
-            return; // already visited
-        }
-
-        // read the plugin dependencies, if any
-        $umdMetadata = "plugins/$plugin/vue/dist/umd.metadata.json";
-
-        $pluginDependencies = [];
-        if (is_file($umdMetadata)) {
-            $pluginDependencies = json_decode(file_get_contents($umdMetadata), true);
-        }
-
-        if (!empty($pluginDependencies['dependsOn'])) {
-            // visit each plugin this one depends on first, so it is loaded first
-            foreach ($pluginDependencies['dependsOn'] as $pluginDependency) {
-                self::visitPlugin($pluginDependency, $plugins, $result);
-            }
-        }
-
-        // add the plugin to the load order after visiting its dependencies
-        $result[] = $plugin;
-    }
 }
diff --git a/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php b/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php
new file mode 100644
index 0000000000..64c7a0f4cb
--- /dev/null
+++ b/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php
@@ -0,0 +1,272 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\AssetManager\UIAssetFetcher;
+
+use Piwik\AssetManager\UIAssetFetcher;
+use Piwik\Config;
+use Piwik\Development;
+use Piwik\Exception\Exception;
+use Piwik\Plugin\Manager;
+
+class PluginUmdAssetFetcher extends UIAssetFetcher
+{
+    /**
+     * @var string
+     */
+    private $requestedChunk;
+
+    /**
+     * @var boolean
+     */
+    private $loadIndividually;
+
+    /**
+     * @var int|null
+     */
+    private $chunkCount;
+
+    public function __construct($plugins, $theme, $chunk, $loadIndividually = null, $chunkCount = null)
+    {
+        parent::__construct($plugins, $theme);
+
+        if ($loadIndividually === null) {
+            $loadIndividually = self::getDefaultLoadIndividually();
+        }
+
+        if ($chunkCount === null) {
+            $chunkCount = self::getDefaultChunkCount();
+        }
+
+        $this->requestedChunk = $chunk;
+        $this->loadIndividually = $loadIndividually;
+        $this->chunkCount = $chunkCount;
+
+        if (!$this->loadIndividually && (!is_int($chunkCount) || $chunkCount <= 0)) {
+            throw new \Exception("Invalid chunk count: $chunkCount");
+        }
+    }
+
+    public function getRequestedChunkOutputFile()
+    {
+        return "asset_manager_chunk.{$this->requestedChunk}.js";
+    }
+
+    /**
+     * @return Chunk[]
+     */
+    public function getChunkFiles()
+    {
+        $allPluginUmds = $this->getAllPluginUmds();
+
+        if ($this->loadIndividually) {
+            return $allPluginUmds;
+        }
+
+        $totalSize = $this->getTotalChunkSize($allPluginUmds);
+
+        $chunkFiles = $this->dividePluginUmdsByChunkCount($allPluginUmds, $totalSize);
+
+        $chunks = [];
+        foreach ($chunkFiles as $index => $jsFiles) {
+            $chunks[] = new Chunk($index, $jsFiles);
+        }
+        return $chunks;
+    }
+
+    private function getTotalChunkSize($allPluginUmds)
+    {
+        $totalSize = 0;
+        foreach ($allPluginUmds as $chunk) {
+            $path = PIWIK_INCLUDE_PATH . '/' . $chunk->getFiles()[0];
+            if (is_file($path)) {
+                $totalSize += filesize($path);
+            }
+        }
+        return $totalSize;
+    }
+
+    private function getAllPluginUmds()
+    {
+        $plugins = self::orderPluginsByPluginDependencies($this->plugins, false);
+
+        $allPluginUmds = [];
+        foreach ($plugins as $plugin) {
+            $pluginDir = self::getRelativePluginDirectory($plugin);
+            $minifiedUmd = "$pluginDir/vue/dist/$plugin.umd.min.js";
+            if (!is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) {
+                continue;
+            }
+
+            $allPluginUmds[] = new Chunk($plugin, [$minifiedUmd]);
+        }
+        return $allPluginUmds;
+    }
+
+    private function dividePluginUmdsByChunkCount($allPluginUmds, $totalSize)
+    {
+        $chunkSizeLimit = floor($totalSize / $this->chunkCount);
+
+        $chunkFiles = [];
+
+        $currentChunkIndex = 0;
+        $currentChunkSize = 0;
+        foreach ($allPluginUmds as $pluginChunk) {
+            $path = PIWIK_INCLUDE_PATH . '/' . $pluginChunk->getFiles()[0];
+            if (!is_file($path)) {
+                continue;
+            }
+
+            $size = filesize($path);
+            $currentChunkSize += $size;
+
+            if ($currentChunkSize > $chunkSizeLimit
+                && !empty($chunkFiles[$currentChunkIndex])
+                && $currentChunkIndex < $this->chunkCount - 1
+            ) {
+                ++$currentChunkIndex;
+                $currentChunkSize = $size;
+            }
+
+            $chunkFiles[$currentChunkIndex][] = $pluginChunk->getFiles()[0];
+        }
+
+        return $chunkFiles;
+    }
+
+    protected function retrieveFileLocations()
+    {
+        if (empty($this->plugins)) {
+            return;
+        }
+
+        if ($this->requestedChunk !== null && $this->requestedChunk !== '') {
+            $chunkFiles = $this->getChunkFiles();
+
+            $foundChunk = null;
+            foreach ($chunkFiles as $chunk) {
+                if ($chunk->getChunkName() == $this->requestedChunk) {
+                    $foundChunk = $chunk;
+                    break;
+                }
+            }
+
+            if (!$foundChunk) {
+                throw new \Exception("Could not find chunk {$this->requestedChunk}");
+            }
+
+            foreach ($foundChunk->getFiles() as $file) {
+                $this->fileLocations[] = $file;
+            }
+
+            return;
+        }
+
+        // either loadFilesIndividually = true, or being called w/ disable_merged_assets=1
+        $this->addUmdFilesIfDetected($this->plugins);
+   }
+
+    private function addUmdFilesIfDetected($plugins)
+    {
+        $plugins = self::orderPluginsByPluginDependencies($plugins, false);
+
+        foreach ($plugins as $plugin) {
+            $pluginDir = self::getRelativePluginDirectory($plugin);
+
+            $devUmd = "$pluginDir/vue/dist/$plugin.development.umd.js";
+            $minifiedUmd = "$pluginDir/vue/dist/$plugin.umd.min.js";
+            $umdSrcFolder = "$pluginDir/vue/src";
+
+            // in case there are dist files but no src files, which can happen during development
+            if (is_dir(PIWIK_INCLUDE_PATH . '/' . $umdSrcFolder)) {
+                if (Development::isEnabled() && is_file(PIWIK_INCLUDE_PATH . '/' . $devUmd)) {
+                    $this->fileLocations[] = $devUmd;
+                } else if (is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) {
+                    $this->fileLocations[] = $minifiedUmd;
+                }
+            }
+        }
+    }
+
+    public static function orderPluginsByPluginDependencies($plugins, $keepUnresolved = true)
+    {
+        $result = [];
+
+        while (!empty($plugins)) {
+            self::visitPlugin(reset($plugins), $keepUnresolved, $plugins, $result);
+        }
+
+        return $result;
+    }
+
+    private static function visitPlugin($plugin, $keepUnresolved, &$plugins, &$result)
+    {
+        // remove the plugin from the array of plugins to visit
+        $index = array_search($plugin, $plugins);
+        if ($index !== false) {
+            unset($plugins[$index]);
+        } else {
+            return; // already visited
+        }
+
+        // read the plugin dependencies, if any
+        $pluginDir = self::getPluginDirectory($plugin);
+        $umdMetadata = "$pluginDir/vue/dist/umd.metadata.json";
+
+        $pluginDependencies = [];
+        if (is_file($umdMetadata)) {
+            $pluginDependencies = json_decode(file_get_contents($umdMetadata), true);
+        }
+
+        if (!empty($pluginDependencies['dependsOn'])) {
+            // visit each plugin this one depends on first, so it is loaded first
+            foreach ($pluginDependencies['dependsOn'] as $pluginDependency) {
+                // check if dependency is not activated
+                if (!in_array($pluginDependency, $plugins)
+                    && !in_array($pluginDependency, $result)
+                ) {
+                    return;
+                }
+
+                self::visitPlugin($pluginDependency, $keepUnresolved, $plugins, $result);
+            }
+        }
+
+        // add the plugin to the load order after visiting its dependencies
+        $result[] = $plugin;
+    }
+
+    protected function getPriorityOrder()
+    {
+        // the JS files are already ordered properly so this result doesn't matter
+        return [];
+    }
+
+    private static function getRelativePluginDirectory($plugin)
+    {
+        $result = self::getPluginDirectory($plugin);
+        $result = str_replace(PIWIK_INCLUDE_PATH . '/', '', $result);
+        return $result;
+    }
+
+    private static function getPluginDirectory($plugin)
+    {
+        return Manager::getInstance()->getPluginDirectory($plugin);
+    }
+
+    public static function getDefaultLoadIndividually()
+    {
+        return (Config::getInstance()->General['assets_umd_load_individually'] ?? 0) == 1;
+    }
+
+    public static function getDefaultChunkCount()
+    {
+        return (int)(Config::getInstance()->General['assets_umd_chunk_count'] ?? 3);
+    }
+}
\ No newline at end of file
diff --git a/core/View.php b/core/View.php
index 624537bc5c..7532a3337e 100644
--- a/core/View.php
+++ b/core/View.php
@@ -363,6 +363,7 @@ class View implements ViewInterface
         $pattern = array(
             '~<script type=[\'"]text/javascript[\'"] src=[\'"]([^\'"]+)[\'"]>~',
             '~<script src=[\'"]([^\'"]+)[\'"] type=[\'"]text/javascript[\'"]>~',
+            '~<script type=[\'"]text/javascript[\'"] src=[\'"]([^\'"]+?chunk=[^\'"]+)[\'"] defer>~',
             '~<link rel=[\'"]stylesheet[\'"] type=[\'"]text/css[\'"] href=[\'"]([^\'"]+)[\'"] ?/?>~',
             // removes the double ?cb= tag
             '~(src|href)=\"index.php\?module=([A-Za-z0-9_]+)&action=([A-Za-z0-9_]+)\?cb=~',
@@ -371,6 +372,7 @@ class View implements ViewInterface
         $replace = array(
             '<script type="text/javascript" src="$1?' . $tagJs . '">',
             '<script type="text/javascript" src="$1?' . $tagJs . '">',
+            '<script type="text/javascript" src="$1&' . $tagJs . '" defer>',
             '<link rel="stylesheet" type="text/css" href="$1?' . $tagCss . '" />',
             '$1="index.php?module=$2&amp;action=$3&amp;cb=',
         );