diff options
author | dizzy <diosmosis@users.noreply.github.com> | 2022-03-04 19:35:44 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-03-04 19:35:44 +0300 |
commit | b774954679f99ef190b9ef71043529563f764946 (patch) | |
tree | acf7041e5865c57841fc09e61369a5c9c8c52385 /core | |
parent | da4fa4175f7cef7cd93d600487fadad34330468a (diff) |
[Vue] Chunk UMD JavaScript into a set of asynchronously loaded files (#18761)
* proof of concept for chunking UMD JavaScript into a set of files that are loaded asynchronously
* take into account alternative plugin directories
* make chunk count and load umds individually configurable (undocumented config) and get to work
* fix a bug and add chunk JS sizes to output of development:compute-js-asset-size
* document
* fill out TODO documentation
* make sure cache buster is added to chunk script srcs
* add some checks in case a chunk does not exist on disk (happens during some tests)
* use realpath on test PIWIK_INCLUDE_PATH so search/replace on paths for relative path will work
* add integration test and get to pass
* fix for when disable_merged_assets=1
* fix condition
* fix ui test failure
* update screenshot
Co-authored-by: sgiehl <stefan@matomo.org>
Diffstat (limited to 'core')
-rw-r--r-- | core/AssetManager.php | 46 | ||||
-rw-r--r-- | core/AssetManager/UIAssetFetcher/Chunk.php | 69 | ||||
-rw-r--r-- | core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php | 63 | ||||
-rw-r--r-- | core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php | 272 | ||||
-rw-r--r-- | core/View.php | 2 |
5 files changed, 386 insertions, 66 deletions
diff --git a/core/AssetManager.php b/core/AssetManager.php index 3e174718e2..c2ad38ef96 100644 --- a/core/AssetManager.php +++ b/core/AssetManager.php @@ -16,6 +16,7 @@ use Piwik\AssetManager\UIAssetCacheBuster; use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher; use Piwik\AssetManager\UIAssetFetcher\StaticUIAssetFetcher; use Piwik\AssetManager\UIAssetFetcher\StylesheetUIAssetFetcher; +use Piwik\AssetManager\UIAssetFetcher\PluginUmdAssetFetcher; use Piwik\AssetManager\UIAssetFetcher; use Piwik\AssetManager\UIAssetMerger\JScriptUIAssetMerger; use Piwik\AssetManager\UIAssetMerger\StylesheetUIAssetMerger; @@ -44,9 +45,11 @@ class AssetManager extends Singleton const CSS_IMPORT_DIRECTIVE = "<link rel=\"stylesheet\" type=\"text/css\" href=\"%s\" />\n"; const JS_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\"></script>\n"; + const JS_DEFER_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\" defer></script>\n"; const GET_CSS_MODULE_ACTION = "index.php?module=Proxy&action=getCss"; const GET_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getCoreJs"; const GET_NON_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getNonCoreJs"; + const GET_JS_UMD_MODULE_ACTION = "index.php?module=Proxy&action=getUmdJs&chunk="; /** * @var UIAssetCacheBuster @@ -149,11 +152,27 @@ class AssetManager extends Singleton } else { $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_CORE_JS_MODULE_ACTION); $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_NON_CORE_JS_MODULE_ACTION); + + $result .= $this->getPluginUmdChunks(); } return $result; } + private function getPluginUmdChunks() + { + $fetcher = $this->getPluginUmdJScriptFetcher(); + + $chunks = $fetcher->getChunkFiles(); + + $result = ''; + foreach ($chunks as $chunk) { + $src = self::GET_JS_UMD_MODULE_ACTION . urlencode($chunk->getChunkName()); + $result .= sprintf(self::JS_DEFER_IMPORT_DIRECTIVE, $src); + } + return $result; + } + /** * Return the base.less compiled to css * @@ -212,7 +231,22 @@ class AssetManager extends Singleton } /** - * @param boolean $core + * Return a chunk JS merged file absolute location. + * If there is none, the generation process will be triggered. + * + * @param string $chunk The name of the chunk. Will either be a plugin name or an integer. + * @return UIAsset + */ + public function getMergedJavaScriptChunk($chunk) + { + $assetFetcher = $this->getPluginUmdJScriptFetcher($chunk); + $outputFile = $assetFetcher->getRequestedChunkOutputFile(); + + return $this->getMergedJavascript($assetFetcher, $this->getMergedUIAsset($outputFile)); + } + + /** + * @param boolean|"all" $core * @return string[] */ public function getLoadedPlugins($core) @@ -223,7 +257,7 @@ class AssetManager extends Singleton $pluginName = $plugin->getPluginName(); $pluginIsCore = Manager::getInstance()->isPluginBundledWithCore($pluginName); - if (($pluginIsCore && $core) || (!$pluginIsCore && !$core)) { + if ($core === 'all' || ($pluginIsCore && $core) || (!$pluginIsCore && !$core)) { $loadedPlugins[] = $pluginName; } } @@ -316,7 +350,8 @@ class AssetManager extends Singleton { return $this->getIndividualJsIncludesFromAssetFetcher($this->getCoreJScriptFetcher()) . - $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher()); + $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher()) . + $this->getIndividualJsIncludesFromAssetFetcher($this->getPluginUmdJScriptFetcher()); } /** @@ -347,6 +382,11 @@ class AssetManager extends Singleton return new JScriptUIAssetFetcher($this->getLoadedPlugins(false), $this->theme); } + protected function getPluginUmdJScriptFetcher($chunk = null) + { + return new PluginUmdAssetFetcher($this->getLoadedPlugins('all'), $this->theme, $chunk); + } + /** * @param string $pluginName * @return boolean diff --git a/core/AssetManager/UIAssetFetcher/Chunk.php b/core/AssetManager/UIAssetFetcher/Chunk.php new file mode 100644 index 0000000000..db42428baa --- /dev/null +++ b/core/AssetManager/UIAssetFetcher/Chunk.php @@ -0,0 +1,69 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\AssetManager\UIAssetFetcher; + +class Chunk +{ + /** + * @var string + */ + private $chunkName; + + /** + * @var string[] + */ + private $files; + + public function __construct($chunkName, $files) + { + $this->chunkName = $chunkName; + $this->files = $files; + } + + /** + * @return string + */ + public function getOutputFile(): string + { + return "asset_manager_chunk.{$this->chunkName}.js"; + } + + /** + * @return string[] + */ + public function getFiles(): array + { + return $this->files; + } + + /** + * @param string[] $files + */ + public function setFiles(array $files): void + { + $this->files = $files; + } + + /** + * @return string + */ + public function getChunkName(): string + { + return $this->chunkName; + } + + /** + * @param string $chunkName + */ + public function setChunkName(string $chunkName): void + { + $this->chunkName = $chunkName; + } +}
\ No newline at end of file diff --git a/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php b/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php index b9a4b3838b..5a31a25505 100644 --- a/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php +++ b/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php @@ -18,7 +18,6 @@ class JScriptUIAssetFetcher extends UIAssetFetcher protected function retrieveFileLocations() { if (!empty($this->plugins)) { - /** * Triggered when gathering the list of all JavaScript files needed by Piwik * and its plugins. @@ -44,8 +43,6 @@ class JScriptUIAssetFetcher extends UIAssetFetcher * @param string[] $jsFiles The JavaScript files to load. */ Piwik::postEvent('AssetManager.getJavaScriptFiles', array(&$this->fileLocations), null, $this->plugins); - - $this->addUmdFilesIfDetected($this->plugins); } $this->addThemeFiles(); @@ -96,64 +93,4 @@ class JScriptUIAssetFetcher extends UIAssetFetcher 'tests/', ); } - - private function addUmdFilesIfDetected($plugins) - { - $plugins = self::orderPluginsByPluginDependencies($plugins); - - foreach ($plugins as $plugin) { - $devUmd = "plugins/$plugin/vue/dist/$plugin.development.umd.js"; - $minifiedUmd = "plugins/$plugin/vue/dist/$plugin.umd.min.js"; - $umdSrcFolder = "plugins/$plugin/vue/src"; - - // in case there are dist files but no src files, which can happen during development - if (is_dir($umdSrcFolder)) { - if (Development::isEnabled() && is_file(PIWIK_INCLUDE_PATH . '/' . $devUmd)) { - $this->fileLocations[$plugin] = $devUmd; - } else if (is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) { - $this->fileLocations[$plugin] = $minifiedUmd; - } - } - } - } - - public static function orderPluginsByPluginDependencies($plugins) - { - $result = []; - - while (!empty($plugins)) { - self::visitPlugin(reset($plugins), $plugins, $result); - } - - return $result; - } - - private static function visitPlugin($plugin, &$plugins, &$result) - { - // remove the plugin from the array of plugins to visit - $index = array_search($plugin, $plugins); - if ($index !== false) { - unset($plugins[$index]); - } else { - return; // already visited - } - - // read the plugin dependencies, if any - $umdMetadata = "plugins/$plugin/vue/dist/umd.metadata.json"; - - $pluginDependencies = []; - if (is_file($umdMetadata)) { - $pluginDependencies = json_decode(file_get_contents($umdMetadata), true); - } - - if (!empty($pluginDependencies['dependsOn'])) { - // visit each plugin this one depends on first, so it is loaded first - foreach ($pluginDependencies['dependsOn'] as $pluginDependency) { - self::visitPlugin($pluginDependency, $plugins, $result); - } - } - - // add the plugin to the load order after visiting its dependencies - $result[] = $plugin; - } } diff --git a/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php b/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php new file mode 100644 index 0000000000..64c7a0f4cb --- /dev/null +++ b/core/AssetManager/UIAssetFetcher/PluginUmdAssetFetcher.php @@ -0,0 +1,272 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\AssetManager\UIAssetFetcher; + +use Piwik\AssetManager\UIAssetFetcher; +use Piwik\Config; +use Piwik\Development; +use Piwik\Exception\Exception; +use Piwik\Plugin\Manager; + +class PluginUmdAssetFetcher extends UIAssetFetcher +{ + /** + * @var string + */ + private $requestedChunk; + + /** + * @var boolean + */ + private $loadIndividually; + + /** + * @var int|null + */ + private $chunkCount; + + public function __construct($plugins, $theme, $chunk, $loadIndividually = null, $chunkCount = null) + { + parent::__construct($plugins, $theme); + + if ($loadIndividually === null) { + $loadIndividually = self::getDefaultLoadIndividually(); + } + + if ($chunkCount === null) { + $chunkCount = self::getDefaultChunkCount(); + } + + $this->requestedChunk = $chunk; + $this->loadIndividually = $loadIndividually; + $this->chunkCount = $chunkCount; + + if (!$this->loadIndividually && (!is_int($chunkCount) || $chunkCount <= 0)) { + throw new \Exception("Invalid chunk count: $chunkCount"); + } + } + + public function getRequestedChunkOutputFile() + { + return "asset_manager_chunk.{$this->requestedChunk}.js"; + } + + /** + * @return Chunk[] + */ + public function getChunkFiles() + { + $allPluginUmds = $this->getAllPluginUmds(); + + if ($this->loadIndividually) { + return $allPluginUmds; + } + + $totalSize = $this->getTotalChunkSize($allPluginUmds); + + $chunkFiles = $this->dividePluginUmdsByChunkCount($allPluginUmds, $totalSize); + + $chunks = []; + foreach ($chunkFiles as $index => $jsFiles) { + $chunks[] = new Chunk($index, $jsFiles); + } + return $chunks; + } + + private function getTotalChunkSize($allPluginUmds) + { + $totalSize = 0; + foreach ($allPluginUmds as $chunk) { + $path = PIWIK_INCLUDE_PATH . '/' . $chunk->getFiles()[0]; + if (is_file($path)) { + $totalSize += filesize($path); + } + } + return $totalSize; + } + + private function getAllPluginUmds() + { + $plugins = self::orderPluginsByPluginDependencies($this->plugins, false); + + $allPluginUmds = []; + foreach ($plugins as $plugin) { + $pluginDir = self::getRelativePluginDirectory($plugin); + $minifiedUmd = "$pluginDir/vue/dist/$plugin.umd.min.js"; + if (!is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) { + continue; + } + + $allPluginUmds[] = new Chunk($plugin, [$minifiedUmd]); + } + return $allPluginUmds; + } + + private function dividePluginUmdsByChunkCount($allPluginUmds, $totalSize) + { + $chunkSizeLimit = floor($totalSize / $this->chunkCount); + + $chunkFiles = []; + + $currentChunkIndex = 0; + $currentChunkSize = 0; + foreach ($allPluginUmds as $pluginChunk) { + $path = PIWIK_INCLUDE_PATH . '/' . $pluginChunk->getFiles()[0]; + if (!is_file($path)) { + continue; + } + + $size = filesize($path); + $currentChunkSize += $size; + + if ($currentChunkSize > $chunkSizeLimit + && !empty($chunkFiles[$currentChunkIndex]) + && $currentChunkIndex < $this->chunkCount - 1 + ) { + ++$currentChunkIndex; + $currentChunkSize = $size; + } + + $chunkFiles[$currentChunkIndex][] = $pluginChunk->getFiles()[0]; + } + + return $chunkFiles; + } + + protected function retrieveFileLocations() + { + if (empty($this->plugins)) { + return; + } + + if ($this->requestedChunk !== null && $this->requestedChunk !== '') { + $chunkFiles = $this->getChunkFiles(); + + $foundChunk = null; + foreach ($chunkFiles as $chunk) { + if ($chunk->getChunkName() == $this->requestedChunk) { + $foundChunk = $chunk; + break; + } + } + + if (!$foundChunk) { + throw new \Exception("Could not find chunk {$this->requestedChunk}"); + } + + foreach ($foundChunk->getFiles() as $file) { + $this->fileLocations[] = $file; + } + + return; + } + + // either loadFilesIndividually = true, or being called w/ disable_merged_assets=1 + $this->addUmdFilesIfDetected($this->plugins); + } + + private function addUmdFilesIfDetected($plugins) + { + $plugins = self::orderPluginsByPluginDependencies($plugins, false); + + foreach ($plugins as $plugin) { + $pluginDir = self::getRelativePluginDirectory($plugin); + + $devUmd = "$pluginDir/vue/dist/$plugin.development.umd.js"; + $minifiedUmd = "$pluginDir/vue/dist/$plugin.umd.min.js"; + $umdSrcFolder = "$pluginDir/vue/src"; + + // in case there are dist files but no src files, which can happen during development + if (is_dir(PIWIK_INCLUDE_PATH . '/' . $umdSrcFolder)) { + if (Development::isEnabled() && is_file(PIWIK_INCLUDE_PATH . '/' . $devUmd)) { + $this->fileLocations[] = $devUmd; + } else if (is_file(PIWIK_INCLUDE_PATH . '/' . $minifiedUmd)) { + $this->fileLocations[] = $minifiedUmd; + } + } + } + } + + public static function orderPluginsByPluginDependencies($plugins, $keepUnresolved = true) + { + $result = []; + + while (!empty($plugins)) { + self::visitPlugin(reset($plugins), $keepUnresolved, $plugins, $result); + } + + return $result; + } + + private static function visitPlugin($plugin, $keepUnresolved, &$plugins, &$result) + { + // remove the plugin from the array of plugins to visit + $index = array_search($plugin, $plugins); + if ($index !== false) { + unset($plugins[$index]); + } else { + return; // already visited + } + + // read the plugin dependencies, if any + $pluginDir = self::getPluginDirectory($plugin); + $umdMetadata = "$pluginDir/vue/dist/umd.metadata.json"; + + $pluginDependencies = []; + if (is_file($umdMetadata)) { + $pluginDependencies = json_decode(file_get_contents($umdMetadata), true); + } + + if (!empty($pluginDependencies['dependsOn'])) { + // visit each plugin this one depends on first, so it is loaded first + foreach ($pluginDependencies['dependsOn'] as $pluginDependency) { + // check if dependency is not activated + if (!in_array($pluginDependency, $plugins) + && !in_array($pluginDependency, $result) + ) { + return; + } + + self::visitPlugin($pluginDependency, $keepUnresolved, $plugins, $result); + } + } + + // add the plugin to the load order after visiting its dependencies + $result[] = $plugin; + } + + protected function getPriorityOrder() + { + // the JS files are already ordered properly so this result doesn't matter + return []; + } + + private static function getRelativePluginDirectory($plugin) + { + $result = self::getPluginDirectory($plugin); + $result = str_replace(PIWIK_INCLUDE_PATH . '/', '', $result); + return $result; + } + + private static function getPluginDirectory($plugin) + { + return Manager::getInstance()->getPluginDirectory($plugin); + } + + public static function getDefaultLoadIndividually() + { + return (Config::getInstance()->General['assets_umd_load_individually'] ?? 0) == 1; + } + + public static function getDefaultChunkCount() + { + return (int)(Config::getInstance()->General['assets_umd_chunk_count'] ?? 3); + } +}
\ No newline at end of file diff --git a/core/View.php b/core/View.php index 624537bc5c..7532a3337e 100644 --- a/core/View.php +++ b/core/View.php @@ -363,6 +363,7 @@ class View implements ViewInterface $pattern = array( '~<script type=[\'"]text/javascript[\'"] src=[\'"]([^\'"]+)[\'"]>~', '~<script src=[\'"]([^\'"]+)[\'"] type=[\'"]text/javascript[\'"]>~', + '~<script type=[\'"]text/javascript[\'"] src=[\'"]([^\'"]+?chunk=[^\'"]+)[\'"] defer>~', '~<link rel=[\'"]stylesheet[\'"] type=[\'"]text/css[\'"] href=[\'"]([^\'"]+)[\'"] ?/?>~', // removes the double ?cb= tag '~(src|href)=\"index.php\?module=([A-Za-z0-9_]+)&action=([A-Za-z0-9_]+)\?cb=~', @@ -371,6 +372,7 @@ class View implements ViewInterface $replace = array( '<script type="text/javascript" src="$1?' . $tagJs . '">', '<script type="text/javascript" src="$1?' . $tagJs . '">', + '<script type="text/javascript" src="$1&' . $tagJs . '" defer>', '<link rel="stylesheet" type="text/css" href="$1?' . $tagCss . '" />', '$1="index.php?module=$2&action=$3&cb=', ); |