Add EncodingService

Signed-off-by: Raul <raul@nextcloud.com>

4 files changed, 114 insertions, 0 deletions

diff --git a/tests/data/big5.txt b/tests/data/big5.txt
new file mode 100644
index 000000000..f507065cd
--- /dev/null
+++ b/tests/data/big5.txt
@@ -0,0 +1,30 @@
+²Ä¤@±ø
+
+¤H¤H¥Í¦Ó¦Û¥Ñ¡A¦b´LÄY©MÅv§Q¤W¤@«ß¥­µ¥¡C¥L­Ì½á¦³²z©Ê©M¨}¤ß¡A¨ÃÀ³¥H¥S§ÌÃö«Yªººë¯«¬Û¹ï«Ý¡C
+²Ä¤G±ø
+
+¤H¤H¦³¸ê®æ¨É¨ü¥»«Å¨¥©Ò¸üªº¤@¤ÁÅv§Q©M¦Û¥Ñ¡A¤£¤ÀºØ±Ú¡B½§¦â¡B©Ê§O¡B»y¨¥¡B©v±Ð¡B¬Fªv©Î¨ä¥L¨£¸Ñ¡B°êÄy©ÎªÀ·|¥X¨­¡B°]²£¡B¥X¥Í©Î¨ä¥L¨­¤Àµ¥¥ô¦ó°Ï§O¡C
+²Ä¤T±ø
+
+¤H¤H¦³Åv¨É¦³¥Í©R¡B¦Û¥Ñ©M¤H¨­¦w¥þ¡C
+²Ä¥|±ø
+
+¥ô¦ó¤H¤£±o¨Ï¬°¥£Áõ©Î¥£§Ð¡F¤@¤Á§Î¦¡ªº¥£Áõ¨î«×©M¥£Áõ¶R½æ¡A§¡À³¤©¥H¸T¤î¡C
+²Ä¤­±ø
+
+¥ô¦ó¤H¤£±o¥[¥H»Å¦D¡A©Î¬I¥H´Ý§Ôªº¡B¤£¤H¹Dªº©Î«V°d©Êªº«Ý¹J©Î¦D»@¡C
+²Ä¤»±ø
+
+¤H¤H¦b¥ô¦ó¦a¤è¦³Åv³Q©Ó»{¦bªk«ß«eªº¤H®æ¡C
+²Ä¤C±ø
+
+ªk«ß¤§«e¤H¤H¥­µ¥¡A¨Ã¦³Åv¨É¨üªk«ßªº¥­µ¥«OÅ@¡A¤£¨ü¥ô¦óª[µø¡C¤H¤H¦³Åv¨É¨ü¥­µ¥«OÅ@¡A¥H§K¨ü¹H¤Ï¥»«Å¨¥ªº¥ô¦óª[µø¦æ¬°¥H¤Îº´°Ê³oºØª[µøªº¥ô¦ó¦æ¬°¤§®`¡C
+²Ä¤K±ø
+
+¥ô¦ó¤H·í¾Ëªk©Îªk«ß©Ò½á¤©¥Lªº°ò¥»Åv§Q¾D¨ü«I®`®É¡A¦³Åv¥Ñ¦X®æªº°ê®aªk®x¹ï³oºØ«I®`¦æ¬°§@¦³®Äªº¸É±Ï¡C
+²Ä¤E±ø
+
+¥ô¦ó¤H¤£±o¥[¥H¥ô·N¶e®·¡B©ë¸T©Î©ñ³v¡C
+²Ä¤Q±ø
+
+¤H¤H§¹¥þ¥­µ¥¦a¦³Åv¥Ñ¤@­Ó¿W¥ß¦ÓµL°¾­Êªºªk®x¶i¦æ¤½¥¿ªº©M¤½¶}ªº¼f°T¡A¥H½T©w¥LªºÅv§Q©M¸q°È¨Ã§P©w¹ï¥L´£¥Xªº¥ô¦ó¦D¨Æ«ü±±¡C
diff --git a/tests/data/gbk.txt b/tests/data/gbk.txt
new file mode 100644
index 000000000..f2846a88d
--- /dev/null
+++ b/tests/data/gbk.txt
@@ -0,0 +1,22 @@
+ÈË ÈË Éú ¶ø ×Ô ÓÉ, ÔÚ ×ð ÑÏ ºÍ Ȩ Àû ÉÏ Ò» ÂÉ Æ½ µÈ¡£ Ëû ÃÇ ¸³ ÓÐ Àí ÐÔ ºÍ Á¼ ÐÄ, ²¢ Ó¦ ÒÔ ÐÖ µÜ ¹Ø ϵ µÄ ¾« Éñ Ïà ¶Ô ´ý¡£
+
+
+Óà ¼ò µ¥ µÄ »° ˵ ¾Í ÊÇ£º
+       ÈË ³ö Éú ʱ ¶¼ ÊÇ ×Ô ÓÉ µÄ£¬ Ó¦ ¸Ã µÃ µ½ ͬ Ñù µÄ ´ý Óö¡£ ÈË ¶¼ ÓÐ Àí ÐÔ ºÍ Á¼ ÐÄ£¬ Ó¦ ¸Ã »¥ Ïà ÓÑ ºÃ Ïà ´¦
+
+
+
+ ÈË ÈË ÓÐ ×Ê ¸ñ Ïí ÓÐ ±¾ Ðû ÑÔ Ëù ÔØ µÄ Ò» ÇРȨ Àû ºÍ ×Ô ÓÉ, ²» ·Ö ÖÖ ×å¡¢ ·ô É«¡¢ ÐÔ ±ð¡¢ Óï ÑÔ¡¢ ×Ú ½Ì¡¢ Õþ ÖÎ »ò Æä Ëû ¼û ½â¡¢ ¹ú ¼® »ò Éç »á ³ö Éí¡¢ ²Æ ²ú¡¢ ³ö Éú »ò Æä Ëû Éí ·Ö µÈ ÈÎ ºÎ Çø ±ð¡£
+²¢ ÇÒ ²» µÃ Òò Ò» ÈË Ëù Êô µÄ ¹ú ¼Ò »ò Áì ÍÁ µÄ Õþ ÖÎ µÄ¡¢ ÐÐ Õþ µÄ »ò Õß ¹ú ¼Ê µÄ µØ λ Ö® ²» ͬ ¶ø ÓÐ Ëù Çø ±ð, ÎÞ ÂÛ ¸Ã Áì ÍÁ ÊÇ ¶À Á¢ Áì ÍÁ¡¢ ÍÐ ¹Ü Áì ÍÁ¡¢ ·Ç ×Ô ÖÎ Áì ÍÁ »ò Õß ´¦ ÓÚ Æä Ëû ÈÎ ºÎ Ö÷ Ȩ ÊÜ ÏÞ ÖÆ µÄ Çé ¿ö Ö® Ï¡£
+
+Óà ¼ò µ¥ µÄ »° ˵ ¾Í ÊÇ£º
+       ¾¡ ¹Ü ÈË ÃÇ ÐÔ ±ð ²» ͬ£¬ Ƥ ·ô ÑÕ É« ²» ͬ£¬ ½² ²» ͬ µÄ Óï ÑÔ£¬ Ïë ²» ͬ µÄ Ê Ç飬 ²Æ ²ú ÓÐ ¶à ÓÐ ÉÙ£¬ Éú »î µÄ Éç »á Ⱥ Ìå ²» ͬ£¬ »ò Õß À´ ×Ô ±ð µÄ ¹ú ¼Ò£¬ µ« ÊÇ Ã¿ ¸ö ÈË ¶¼ ÓÐ ÒÔ Ï Ȩ Àû¡£ Äã Ëù ÔÚ µÄ ¹ú ¼Ò ÊÇ ·ñ ¶À Á¢£¬ ¶Ô Õâ Щ Ȩ Àû ²» Ó¦ ¸Ã ÓÐ Ó° Ïì¡£
+
+
+
+ÈË ÈË ÓРȨ Ïí ÓÐ Éú Ãü¡¢ ×Ô ÓÉ ºÍ ÈË Éí °² È«¡£
+
+
+Óà ¼ò µ¥ µÄ »° ˵ ¾Í ÊÇ£º
+       Äã ÓÐ »î ×Å µÄ Ȩ Àû£¬ ÓÐ ×Ô ÓÉ °² È« µØ Éú »î µÄ Ȩ Àû¡£
+
diff --git a/tests/data/iso-8859.txt b/tests/data/iso-8859.txt
new file mode 100644
index 000000000..7f1b95575
--- /dev/null
+++ b/tests/data/iso-8859.txt
@@ -0,0 +1,6 @@
+Todos os seres humanos nascem livres e iguais em dignidade e em direitos. Dotados de razão e de consciência, devem agir uns para com os outros em espírito de fraternidade. 
+
+Alle menneske er fødde til fridom og med same menneskeverd og menneskerettar. Dei har fått fornuft og samvit og skal leve med kvarandre som brør.
+
+Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité. 
+
diff --git a/tests/unit/Service/EncodingServiceTest.php b/tests/unit/Service/EncodingServiceTest.php
new file mode 100644
index 000000000..8458aaae9
--- /dev/null
+++ b/tests/unit/Service/EncodingServiceTest.php
@@ -0,0 +1,56 @@
+<?php
+
+namespace OCA\Text\Service;
+
+use Test\TestCase;
+
+class LabelServiceTest extends TestCase {
+	private $encodingService;
+
+	protected function setUp(): void {
+		parent::setUp();
+		$this->encodingService = new EncodingService();
+	}
+
+	/**
+	 * Attempt to decode the file using the default decoding order.
+	 * For files with encodings not included in the COMMON_ENCODINGS array encoding to UTF-8 will fail.
+	 * @dataProvider dataFileEncodings
+	 */
+	public function testDefault(string $file, string $encoding) {
+		$utf8_string = $this->encodingService->encodeToUtf8(file_get_contents($file));
+
+		// If encoding is not part of the default encodings we can expect it to fail
+		// It might still succeed because encoding detection is not precise.
+		if (!$utf8_string && !in_array($encoding, EncodingService::COMMON_ENCODINGS, true)) {
+			return;
+		}
+
+		$this->assertNotNull($utf8_string);
+		$this->assertNotFalse(mb_detect_encoding($utf8_string, 'UTF-8', true));
+	}
+
+	/**
+	 * Includes the encoding of the file in the detection order config value.
+	 * This means that all files should be successfully encoded to UTF-8.
+	 * @dataProvider dataFileEncodings
+	 */
+	public function testCustomOrder(string $file, string $encoding) {
+		$original_order = mb_detect_order();
+		$this->assertNotFalse(mb_detect_order($encoding));
+
+		$utf8_string = $this->encodingService->encodeToUtf8(file_get_contents($file));
+		$this->assertNotNull($utf8_string);
+		$this->assertNotFalse(mb_detect_encoding($utf8_string, 'UTF-8', true));
+
+		mb_detect_order($original_order);
+	}
+
+	public function dataFileEncodings(): array {
+		return [
+			['./tests/data/iso-8859.txt', 'ISO-8859-1'],
+			['./tests/data/big5.txt', 'BIG-5'],
+			['./tests/data/gbk.txt', 'GBK']
+		];
+	}
+}