Test for PR #9486, Validate allowed TLDs with current IANA list

2 files changed, 185 insertions, 12 deletions

diff --git a/libs/Zend/Validate/Hostname.php b/libs/Zend/Validate/Hostname.php
index cd5db5abcb..5dc6e818d7 100644
--- a/libs/Zend/Validate/Hostname.php
+++ b/libs/Zend/Validate/Hostname.php
@@ -201,16 +201,7 @@ class Zend_Validate_Hostname extends Zend_Validate_Abstract
         'villas', 'vin', 'vip', 'virgin', 'vision', 'vista', 'vistaprint', 'viva', 'vlaanderen', 'vn', 'vodka', 'vote', 
         'voting', 'voto', 'voyage', 'vu', 'wales', 'walter', 'wang', 'wanggou', 'watch', 'watches', 'webcam', 'weber', 
         'website', 'wed', 'wedding', 'weir', 'wf', 'whoswho', 'wien', 'wiki', 'williamhill', 'win', 'windows', 'wine', 
-        'wme', 'work', 'works', 'world', 'ws', 'wtc', 'wtf', 'xbox', 'xerox', 'xin', 'कॉम', '佛山', '慈善', '集团', 
-        '在线', '한국', '点看', 'คอม', 'ভারত', '八卦', 'موقع', '公益', '公司', '移动', '我爱你', 'москва', 'қаз', 'онлайн', 
-        'сайт', 'срб', 'бел', 'קום', '时尚', '淡马锡', 'орг', 'नेट', '삼성', 'சிங்கப்பூர்', '商标', '商店', '商城', 'дети', 
-        'мкд', 'ポイント', '新闻', '工行', 'كوم', '中文网', '中信', '中国', '中國', '娱乐', '谷歌', 'భారత్', 'ලංකා', 'ભારત', 
-        'भारत', '网店', 'संगठन', '餐厅', '网络', 'ком', 'укр', '香港', '诺基亚', '飞利浦', '台湾', '台灣', '手表', '手机', 
-        'мон', 'الجزائر', 'عمان', 'ارامكو', 'ایران', 'امارات', 'بازار', 'الاردن', 'موبايلي', 'بھارت', 'المغرب', 
-        'السعودية', 'سودان', 'همراه', 'عراق', 'مليسيا', '닷컴', '政府', 'شبكة', 'بيتك', 'გე', '机构', '组织机构', 
-        '健康', 'ไทย', 'سورية', 'рус', 'рф', '珠宝', 'تونس', '大拿', 'みんな', 'グーグル', 'ελ', '世界', 'ਭਾਰਤ', '网址', 
-        '닷넷', 'コム', '游戏', 'vermögensberater', 'vermögensberatung', '企业', '信息', 'مصر', 'قطر', '广东', 'இலங்கை', 
-        'இந்தியா', 'հայ', '新加坡', 'فلسطين', '政务', 'xperia', 'xxx', 'xyz', 'yachts', 'yamaxun', 'yandex', 'ye', 
+        'wme', 'work', 'works', 'world', 'ws', 'wtc', 'wtf', 'xbox', 'xerox', 'xin', 'xperia', 'xxx', 'xyz', 'yachts', 'yamaxun', 'yandex', 'ye',
         'yodobashi', 'yoga', 'yokohama', 'youtube', 'yt', 'za', 'zara', 'zero', 'zip', 'zm', 'zone', 'zuerich', 'zw'
     );
 
@@ -371,9 +362,134 @@ class Zend_Validate_Hostname extends Zend_Validate_Abstract
         'ایران' => array(1 => '/^[\x{0621}-\x{0624}\x{0626}-\x{063A}\x{0641}\x{0642}\x{0644}-\x{0648}\x{067E}\x{0686}\x{0698}\x{06A9}\x{06AF}\x{06CC}\x{06F0}-\x{06F9}]{1,30}$/iu'),
         '中国' => 'Hostname/Cn.php',
         '公司' => 'Hostname/Cn.php',
-        '网络' => 'Hostname/Cn.php'
+        '网络' => 'Hostname/Cn.php', 
+        'कॉम' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'セール' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '佛山' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '慈善' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '集团' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '在线' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '한국' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '点看' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'คอม' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ভারত' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '八卦' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'موقع' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '公益' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '公司' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '移动' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '我爱你' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'москва' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'қаз' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'онлайн' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'сайт' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '联通' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'срб' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'бел' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'קום' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '时尚' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '淡马锡' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ファッション' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'орг' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'नेट' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ストア' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '삼성' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'சிங்கப்பூர்' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '商标' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '商店' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '商城' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'дети' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'мкд' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ею' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ポイント' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '新闻' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '工行' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'كوم' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '中文网' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '中信' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '中国' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '中國' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '娱乐' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '谷歌' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'భారత్' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ලංකා' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '购物' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'クラウド' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ભારત' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'भारत' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '网店' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'संगठन' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '餐厅' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '网络' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ком' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'укр' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '香港' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '诺基亚' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '食品' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '飞利浦' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '台湾' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '台灣' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '手表' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '手机' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'мон' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'الجزائر' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'عمان' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ارامكو' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ایران' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'امارات' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'بازار' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'الاردن' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'موبايلي' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'بھارت' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'المغرب' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'السعودية' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'سودان' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'همراه' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'عراق' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'مليسيا' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '澳門' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '닷컴' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '政府' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'شبكة' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'بيتك' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'გე' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '机构' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '组织机构' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '健康' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ไทย' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'سورية' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'рус' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'рф' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '珠宝' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'تونس' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '大拿' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'みんな' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'グーグル' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ελ' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '世界' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '書籍' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'ਭਾਰਤ' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '网址' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '닷넷' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'コム' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '游戏' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'VERMöGENSBERATER' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'VERMöGENSBERATUNG' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '企业' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '信息' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '嘉里大酒店' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'مصر' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'قطر' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '广东' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'இலங்கை' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'இந்தியா' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'հայ' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '新加坡' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        'فلسطين' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu'),
+        '政务' => array(1 => '/^[\p{L}\p{M}]{1,63}$/iu')
     );
 
+
     protected $_idnLength = array(
         'BIZ' => array(5 => 17, 11 => 15, 12 => 20),
         'CN'  => array(1 => 20),
@@ -744,7 +860,7 @@ class Zend_Validate_Hostname extends Zend_Validate_Abstract
 
         // Check for URI Syntax (RFC3986)
         if ($this->_options['allow'] & self::ALLOW_URI) {
-            if (preg_match("/^([a-zA-Z0-9-._~!$&\'()*+,;=]|%[[:xdigit:]]{2}){1,254}$/i", $value)) {
+            if (preg_match("/^([a-zA-Z0-9-._~!$&'()*+,;=]|%[[:xdigit:]]{2}){1,254}$/i", $value)) {
                 return true;
             } else {
                 $this->_error(self::INVALID_URI);
diff --git a/tests/PHPUnit/Integration/EmailValidatorTest.php b/tests/PHPUnit/Integration/EmailValidatorTest.php
index 4b2e215506..262baab9e0 100644
--- a/tests/PHPUnit/Integration/EmailValidatorTest.php
+++ b/tests/PHPUnit/Integration/EmailValidatorTest.php
@@ -8,6 +8,7 @@
 
 namespace Piwik\Tests\Integration;
 
+use Piwik\Http;
 use Piwik\Piwik;
 
 /**
@@ -20,6 +21,62 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
         return Piwik::isValidEmailString($email);
     }
 
+    private function getAllTlds()
+    {
+        /** @var array $response */
+        $response = \Piwik\Http::sendHttpRequest("http://data.iana.org/TLD/tlds-alpha-by-domain.txt", 30, null, null, null, null, null, true);
+
+        $this->assertEquals("200", $response['status']);
+
+        $tlds = explode("\n", $response['data']);
+        foreach ($tlds as $key => $tld) {
+            if (strpos($tld, '#') !== false || $tld == "") {
+                unset($tlds[$key]);
+            }
+        }
+        return $tlds;
+    }
+
+    public function test_allCurrentTlds(){
+        $tlds = $this->getAllTlds();
+        if (count($tlds) === 0) {
+            $this->markTestSkipped("Couldn't get TLD list");
+        }
+
+        foreach ($tlds as $key => $tld) {
+            if (strpos(mb_strtolower($tld), 'xn--') !== 0) {
+                $tld = mb_strtolower($tld);
+            }
+            $this->assertTrue(
+                $this->isValid('test@example.' . idn_to_utf8($tld))
+            );
+        }
+    }
+
+    public function test_invalidTld(){
+        $tlds = [
+            strval(bin2hex(openssl_random_pseudo_bytes(64))), //generates 128 bit length string
+            '-tld-cannot-start-from-hypen',
+            'ąęśćżźł-there-is-no-such-idn',
+            'xn--fd67as67fdsa', //no such idn punycode
+            '!@#-inavlid-chars-in-tld',
+            'no spaces in tld allowed',
+            'no--double--hypens--allowed'
+        ];
+        if (count($tlds) === 0) {
+            $this->markTestSkipped("Couldn't get TLD list");
+        }
+
+        foreach ($tlds as $key => $tld) {
+            if (strpos(mb_strtolower($tld), 'xn--') !== 0) {
+                $tld = mb_strtolower($tld);
+            }
+            $this->assertFalse(
+                $this->isValid('test@example.' . idn_to_utf8($tld))
+            );
+        }
+    }
+
     public function test_isValid_validStandard()
     {
         $this->assertTrue($this->isValid('test@example.com'));