Merge pull request #25529 from nextcloud/fix-non-lgc-glyphs-in-avatars-and-txt-file-previews

Fix non LGC glyphs in avatars and txt file previews

3 files changed, 379 insertions, 0 deletions

diff --git a/build/merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch b/build/merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
new file mode 100644
index 00000000000..b8f39d4b9ce
--- /dev/null
+++ b/build/merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
@@ -0,0 +1,230 @@
+diff --git a/nototools/merge_noto.py b/nototools/merge_noto.py
+index 17c07ed..029845a 100755
+--- a/nototools/merge_noto.py
++++ b/nototools/merge_noto.py
+@@ -34,7 +34,7 @@ def make_puncless_font_name(script):
+     return make_font_name(script).replace(" ", "").replace("-", "")
+ 
+ 
+-def make_font_file_name(script, weight, directory="individual/unhinted"):
++def make_font_file_name(script, weight, directory="individual/hinted"):
+     filename = "%s/%s-%s.ttf" % (directory, make_puncless_font_name(script), weight)
+     return filename
+ 
+@@ -85,6 +85,11 @@ SCRIPT_TO_OPENTYPE_SCRIPT_TAG = {
+     "Cuneiform": "xsux",
+     "Cypriot": "cprt",
+     "Yi": "yi  ",
++    "AnatolianHieroglyphs":"hluw",
++    "Bamum": "bamu",
++    "NewTaiLue": "talu",
++    "Tagbanwa": "tagb",
++    "Thaana": "thaa",
+ }
+ 
+ 
+@@ -135,96 +140,129 @@ def add_gsub_to_font(fontfile):
+ 
+ def main():
+     merge_table = {
+-        "Historic": [
++        # Use a single file with all the fonts copied from merge_fonts.py.
++        "": [  # LGC,
++            "Adlam",
++            "AdlamUnjoined",
++            "AnatolianHieroglyphs",
++            "Arabic",
++            "ArabicUI",
++            "Armenian",
+             "Avestan",
+-            "Carian",
+-            "Egyptian Hieroglyphs",
+-            "Imperial Aramaic",
+-            "Pahlavi",  # Should be 'Inscriptional Pahlavi',
+-            "Parthian",  # Should be 'Inscriptional Parthian',
+-            "Linear B",
+-            "Lycian",
+-            "Lydian",
+-            "Mandaic",
+-            "Old Persian",
+-            "Old South Arabian",
+-            "Old Turkic",
+-            "Osmanya",
+-            "Phags-Pa",
+-            "Phoenician",
+-            "Samaritan",
+-            "Sumero-Akkadian Cuneiform",
+-            "Ugaritic",
+-        ],
+-        "South Asian": [
+-            "Devanagari",
++            "Balinese",
++            "Bamum",
++            "Batak",
+             "Bengali",
+-            "Gurmukhi",
+-            "Gujarati",
+-            "Oriya",
+-            "Tamil",
+-            "Telugu",
+-            "Kannada",
+-            "Malayalam",
+-            "Sinhala",
+-            "Thaana",
++            "BengaliUI",
+             "Brahmi",
+-            "Kaithi",
+-            "Kharoshthi",  # Move to Historic?
+-            "Lepcha",
+-            "Limbu",
+-            "Meetei Mayek",
+-            "Ol Chiki",
+-            "Saurashtra",
+-            "Syloti Nagri",
+-        ],
+-        "Southeast Asian": [
+-            "Thai",
+-            "Lao",
+-            "Khmer",
+-            "Batak",
+             "Buginese",
+             "Buhid",
++            "CJKjp-Regular.otf",
++            "CJKkr-Regular.otf",
++            "CJKsc-Regular.otf",
++            "CJKtc-Regular.otf",
++            "CanadianAboriginal",
++            "Carian",
++            "Chakma",
+             "Cham",
+-            "Hanunoo",
+-            "Javanese",
+-            "Kayah Li",
+-            "New Tai Lue",
+-            "Rejang",
+-            "Sundanese",
+-            "Tagalog",
+-            "Tagbanwa",
+-            "Tai Le",
+-            "Tai Tham",
+-            "Tai Viet",
+-        ],
+-        "": [  # LGC,
+-            "Armenian",
+-            "Bamum",
+-            "Canadian Aboriginal",
+             "Cherokee",
+             "Coptic",
+-            "Cypriot Syllabary",
++            "Cuneiform",
++            "Cypriot",
+             "Deseret",
++            "Devanagari",
++            "DevanagariUI",
++            "Display",
++            "EgyptianHieroglyphs",
+             "Ethiopic",
+             "Georgian",
+             "Glagolitic",
+             "Gothic",
++            "Gujarati",
++            "GujaratiUI",
++            "Gurmukhi",
++            "GurmukhiUI",
++            "Hanunoo",
+             "Hebrew",
++            "ImperialAramaic",
++            "InscriptionalPahlavi",
++            "InscriptionalParthian",
++            "Javanese",
++            "Kaithi",
++            "Kannada",
++            "KannadaUI",
++            "KayahLi",
++            "Kharoshthi",
++            "Khmer",
++            "KhmerUI",
++            "Lao",
++            "LaoUI",
++            "Lepcha",
++            "Limbu",
++            "LinearB",
+             "Lisu",
++            "Lycian",
++            "Lydian",
++            "Malayalam",
++            "MalayalamUI",
++            "Mandaic",
++            "MeeteiMayek",
++            #"NotoSansMongolian",
++            "Mono",
++            "MonoCJKjp-Regular.otf",
++            "MonoCJKkr-Regular.otf",
++            "MonoCJKsc-Regular.otf",
++            "MonoCJKtc-Regular.otf",
++            "Myanmar",
++            "MyanmarUI",
+             "NKo",
++            "NewTaiLue",
+             "Ogham",
+-            "Old Italic",
++            "OlChiki",
++            "OldItalic",
++            "OldPersian",
++            "OldSouthArabian",
++            "OldTurkic",
++            "Oriya",
++            "OriyaUI",
++            "Osage",
++            "Osmanya",
++            "PhagsPa",
++            "Phoenician",
++            "Rejang",
+             "Runic",
++            "Samaritan",
++            "Saurashtra",
+             "Shavian",
++            "Sinhala",
++            "SinhalaUI",
++            "Sundanese",
++            "SylotiNagri",
++            "Symbols",
++            "Symbols2",
++            "SyriacEastern",
++            "SyriacEstrangela",
++            "SyriacWestern",
++            "Tagalog",
++            "Tagbanwa",
++            "TaiLe",
++            "TaiTham",
++            "TaiViet",
++            "Tamil",
++            "TamilUI",
++            "Telugu",
++            "TeluguUI",
++            "Thaana",
++            "Thai",
++            "ThaiUI",
++            "Tibetan",
+             "Tifinagh",
++            "Ugaritic",
+             "Vai",
++            "Yi",
+         ],
+     }
+ 
+-    add_ui_alternative(merge_table, "South Asian")
+-    add_ui_alternative(merge_table, "Southeast Asian")
+-
+     for merge_target in sorted(merge_table):
+         for weight in ["Regular", "Bold"]:
+             merger = merge.Merger()
+@@ -261,7 +299,7 @@ def main():
+                     name_record.string = name.encode("UTF-16BE")
+ 
+             font.save(
+-                make_font_file_name(merge_target, weight, directory="combined/unhinted")
++                make_font_file_name(merge_target, weight, directory="combined/hinted")
+             )
+ 
+ 
diff --git a/build/merge-font-noto.sh b/build/merge-font-noto.sh
new file mode 100755
index 00000000000..ee4831dac74
--- /dev/null
+++ b/build/merge-font-noto.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+# @copyright Copyright (c) 2021, Daniel Calviño Sánchez (danxuliu@gmail.com)
+#
+# @license GNU AGPL version 3 or any later version
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Helper script to merge several Noto fonts in a single TTF file.
+#
+# The "Noto Sans" font (https://www.google.com/get/noto) only includes a subset
+# of all the available glyphs in the Noto fonts. This scripts uses
+# "merge_noto.py" from the Noto Tools package to add other scripts, like Arabic,
+# Devanagari or Hebrew.
+#
+# "merge_noto.py" originally merges the fonts by region. However it was adjusted
+# to merge "all" the fonts in a single file, like done by "merge_fonts.py". The
+# reason to use "merge_noto.py" instead of "merge_fonts.py" is that
+# "merge_noto.py" merges regular and bold fonts, which are both needed in
+# Nextcloud. "merge_fonts.py" only merges regular fonts, and adjusting it to
+# handle bold fonts too would have been more work than adjusting
+# "merge_noto.py".
+#
+# Please note that, due to technical limitations of the TTF format (a single
+# file can not have more than 65535 glyphs) the merged file does not include any
+# Chinese, Japanese or Korean glyph (the Noto CJK files already use all the
+# slots). In fact, it seems that it can not include either all the glyphs from
+# all the non CJK Noto fonts, so it merges only those predefined in the
+# "merge_fonts.py" script (as it is a larger set than the original one in
+# "merge_noto.py").
+#
+# Also please note that merging the fonts is a slow process and it can take a
+# while (from minutes to hours, depending on the system).
+#
+# To perform its job, the script requires the "docker" command to be available.
+#
+# The Docker Command Line Interface (the "docker" command) requires special
+# permissions to talk to the Docker daemon, and those permissions are typically
+# available only to the root user. Please see the Docker documentation to find
+# out how to give access to a regular user to the Docker daemon:
+# https://docs.docker.com/engine/installation/linux/linux-postinstall/
+#
+# Note, however, that being able to communicate with the Docker daemon is the
+# same as being able to get root privileges for the system. Therefore, you must
+# give access to the Docker daemon (and thus run this script as) ONLY to trusted
+# and secure users:
+# https://docs.docker.com/engine/security/security/#docker-daemon-attack-surface
+
+# Stops the container started by this script.
+function cleanUp() {
+	# Disable (yes, "+" disables) exiting immediately on errors to ensure that
+	# all the cleanup commands are executed (well, no errors should occur during
+	# the cleanup anyway, but just in case).
+	set +o errexit
+
+	echo "Cleaning up"
+	docker rm --volumes --force $DOCKER_CONTAINER_ID
+}
+
+# Exit immediately on errors.
+set -o errexit
+
+# Execute cleanUp when the script exits, either normally or due to an error.
+trap cleanUp EXIT
+
+# Ensure working directory is script directory, as some actions (like copying
+# the patches to the container) expect that.
+cd "$(dirname $0)"
+
+# python:3.9 can not be used, as one of the requeriments of Noto Tools
+# (pyclipper) fails to build.
+#
+# The container exits immediately if no command is given, so a Bash session
+# is created to prevent that.
+DOCKER_CONTAINER_ID=`docker run --rm --detach --interactive --tty python:3.8-slim bash`
+
+# Install required dependencies.
+docker exec $DOCKER_CONTAINER_ID apt-get update
+docker exec $DOCKER_CONTAINER_ID apt-get install -y git gcc g++ libjpeg-dev zlib1g-dev wget
+
+# Install Noto Tools in the container.
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID git clone https://github.com/googlefonts/nototools
+docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID git checkout 76b29f8f8f9b
+docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID pip install --requirement requirements.txt
+docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID pip install --editable .
+
+# As Noto Tools were installed as "editable" the scripts can be patched after
+# installation.
+docker cp merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch $DOCKER_CONTAINER_ID:/tmp/nototools/merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
+docker exec --workdir /tmp/nototools --interactive $DOCKER_CONTAINER_ID patch --strip 1 < merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
+
+# Get Noto fonts.
+#
+# Phase 2 Noto fonts use 2048 units per em, while phase 3 Noto fonts use 1000*.
+# Currently the fonts in the released package** (apparently from 2017-10-25) are
+# a mix of both, but fonts with different units per em can not be merged***.
+# However, the fonts in the Git repository, although not released yet, are all
+# using 1000 units per em already, so those are the ones merged.
+#
+# *https://github.com/googlefonts/noto-fonts/issues/908#issuecomment-298687906.
+# **https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
+# ***https://fonttools.readthedocs.io/en/latest/merge.html
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID wget https://github.com/googlefonts/noto-fonts/archive/v20201206-phase3.tar.gz
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID tar -xzf v20201206-phase3.tar.gz
+
+# noto-fonts in Git and snapshots of Git (like the package used) have a
+# subdirectory for each font, but "merge_noto.py" expects to find all the fonts
+# in a single directory, so the structure needs to be "flattened".
+#
+# Hinted fonts* adapt better to being rendered in different sizes. The full
+# package in https://www.google.com/get/noto/ includes only unhinted fonts
+# (according to its name**, I have not actually verified the fonts themselves),
+# while the individual fonts listed below in the page are a mix of hinted and
+# unhinted fonts. However, the Git directory has hinted versions of all fonts,
+# so those are the ones merged (maybe there is a good reason not to merge hinted
+# fonts, but seems to work :-P).
+#
+# *https://en.wikipedia.org/wiki/Font_hinting
+# **https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID mkdir --parent individual/hinted
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID find noto-fonts-20201206-phase3/hinted/ttf -iname "NotoSans*Regular.ttf" -exec mv {} individual/hinted/ \;
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID find noto-fonts-20201206-phase3/hinted/ttf -iname "NotoSans*Bold.ttf" -exec mv {} individual/hinted/ \;
+
+# Merge the fonts.
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID mkdir --parent combined/hinted
+docker exec --workdir /tmp $DOCKER_CONTAINER_ID merge_noto.py
+
+# Copy resulting files.
+#
+# Noto fonts, as well as the merged files, are licensed under the SIL Open Font
+# License: https://scripts.sil.org/OFL
+docker cp $DOCKER_CONTAINER_ID:/tmp/combined/hinted/NotoSans-Regular.ttf ../core/fonts/NotoSans-Regular.ttf
+docker cp $DOCKER_CONTAINER_ID:/tmp/combined/hinted/NotoSans-Bold.ttf ../core/fonts/NotoSans-Bold.ttf
diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml
index 3c9f033d573..7510cb0c6f2 100644
--- a/build/psalm-baseline.xml
+++ b/build/psalm-baseline.xml
@@ -2517,6 +2517,11 @@
       <code>section</code>
     </UndefinedInterfaceMethod>
   </file>
+  <file src="core/Command/Preview/ResetRenderedTexts.php">
+    <InvalidReturnStatement occurrences="1">
+      <code>[]</code>
+    </InvalidReturnStatement>
+  </file>
   <file src="core/Command/Upgrade.php">
     <InvalidScalarArgument occurrences="11">
       <code>0</code>