Merge pull request #10169 from SilverFire/mb-function-encoding

Normalized mb_* function calls to use Yii::$app->charset

framework/CHANGELOG.md

@@ -38,6 +38,7 @@ Yii Framework 2 Change Log
 - Bug #10101: Fixed assignments saving on role removing in `\yii\rbac\PhpManager` (rezident1307)
 - Bug #10142: Fixed `yii\validators\EmailValidator` to check the length of email properly (silverfire)
 - Bug: Fixed generation of canonical URLs for `ViewAction` pages (samdark)
+- Bug: Fixed `mb_*` functions calls to use `UTF-8` or `Yii::$app->charset` (silverfire)
 - Enh #3506: Added `\yii\validators\IpValidator` to perform validation of IP addresses and subnets (SilverFire, samdark)
 - Enh #5146: Added `\yii\i18n\Formatter::asDuration()` method (nineinchnick, SilverFire)
 - Enh #7341: Client validation now skips disabled inputs (SamMousa)

framework/helpers/BaseFileHelper.php

@@ -189,7 +189,7 @@ class BaseFileHelper
     public static function getExtensionsByMimeType($mimeType, $magicFile = null)
     {
         $mimeTypes = static::loadMimeTypes($magicFile);
-        return array_keys($mimeTypes, mb_strtolower($mimeType, 'utf-8'), true);
+        return array_keys($mimeTypes, mb_strtolower($mimeType, 'UTF-8'), true);
     }
 
     private static $_mimeTypes = [];

framework/i18n/MessageFormatter.php

@@ -7,6 +7,7 @@
 
 namespace yii\i18n;
 
+use Yii;
 use yii\base\Component;
 use yii\base\NotSupportedException;
 
@@ -274,19 +275,20 @@ class MessageFormatter extends Component
      */
     private static function tokenizePattern($pattern)
     {
+        $charset = Yii::$app ? Yii::$app->charset : 'UTF-8';
         $depth = 1;
-        if (($start = $pos = mb_strpos($pattern, '{')) === false) {
+        if (($start = $pos = mb_strpos($pattern, '{', 0, $charset)) === false) {
             return [$pattern];
         }
-        $tokens = [mb_substr($pattern, 0, $pos)];
+        $tokens = [mb_substr($pattern, 0, $pos, $charset)];
         while (true) {
-            $open = mb_strpos($pattern, '{', $pos + 1);
+            $open = mb_strpos($pattern, '{', $pos + 1, $charset);
-            $close = mb_strpos($pattern, '}', $pos + 1);
+            $close = mb_strpos($pattern, '}', $pos + 1, $charset);
             if ($open === false && $close === false) {
                 break;
             }
             if ($open === false) {
-                $open = mb_strlen($pattern);
+                $open = mb_strlen($pattern, $charset);
             }
             if ($close > $open) {
                 $depth++;
@@ -296,9 +298,9 @@ class MessageFormatter extends Component
                 $pos = $close;
             }
             if ($depth === 0) {
-                $tokens[] = explode(',', mb_substr($pattern, $start + 1, $pos - $start - 1), 3);
+                $tokens[] = explode(',', mb_substr($pattern, $start + 1, $pos - $start - 1, $charset), 3);
                 $start = $pos + 1;
-                $tokens[] = mb_substr($pattern, $start, $open - $start);
+                $tokens[] = mb_substr($pattern, $start, $open - $start, $charset);
                 $start = $open;
             }
         }
@@ -321,7 +323,7 @@ class MessageFormatter extends Component
     {
         // parsing pattern based on ICU grammar:
         // http://icu-project.org/apiref/icu4c/classMessageFormat.html#details
-
+        $charset = Yii::$app ? Yii::$app->charset : 'UTF-8';
         $param = trim($token[0]);
         if (isset($args[$param])) {
             $arg = $args[$param];
@@ -391,11 +393,11 @@ class MessageFormatter extends Component
                     $selector = trim($plural[$i++]);
 
                     if ($i == 1 && strncmp($selector, 'offset:', 7) === 0) {
-                        $offset = (int) trim(mb_substr($selector, 7, ($pos = mb_strpos(str_replace(["\n", "\r", "\t"], ' ', $selector), ' ', 7)) - 7));
+                        $offset = (int) trim(mb_substr($selector, 7, ($pos = mb_strpos(str_replace(["\n", "\r", "\t"], ' ', $selector), ' ', 7, $charset)) - 7, $charset));
-                        $selector = trim(mb_substr($selector, $pos + 1));
+                        $selector = trim(mb_substr($selector, $pos + 1, null, $charset));
                     }
                     if ($message === false && $selector === 'other' ||
-                        $selector[0] === '=' && (int) mb_substr($selector, 1) === $arg ||
+                        $selector[0] === '=' && (int) mb_substr($selector, 1, null, $charset) === $arg ||
                         $selector === 'one' && $arg - $offset == 1
                     ) {
                         $message = implode(',', str_replace('#', $arg - $offset, $plural[$i]));

framework/validators/FileValidator.php

@@ -315,7 +315,7 @@ class FileValidator extends Validator
      */
     protected function validateExtension($file)
     {
-        $extension = mb_strtolower($file->extension, 'utf-8');
+        $extension = mb_strtolower($file->extension, 'UTF-8');
 
         if ($this->checkExtensionByMimeType) {
 

framework/web/AssetManager.php

@@ -351,9 +351,9 @@ class AssetManager extends Component
             $asset = $bundle->sourcePath . '/' . $asset;
         }
 
-        $n = mb_strlen($asset);
+        $n = mb_strlen($asset, Yii::$app->charset);
         foreach ($this->assetMap as $from => $to) {
-            $n2 = mb_strlen($from);
+            $n2 = mb_strlen($from, Yii::$app->charset);
             if ($n2 <= $n && substr_compare($asset, $from, $n - $n2, $n2) === 0) {
                 return $to;
             }

framework/web/ErrorHandler.php

@@ -391,8 +391,8 @@ class ErrorHandler extends \yii\base\ErrorHandler
                 $args[$key] = '<span class="keyword">' . ($value ? 'true' : 'false') . '</span>';
             } elseif (is_string($value)) {
                 $fullValue = $this->htmlEncode($value);
-                if (mb_strlen($value, 'utf8') > 32) {
+                if (mb_strlen($value, Yii::$app->charset) > 32) {
-                    $displayValue = $this->htmlEncode(mb_substr($value, 0, 32, 'utf8')) . '...';
+                    $displayValue = $this->htmlEncode(mb_substr($value, 0, 32, Yii::$app->charset)) . '...';
                     $args[$key] = "<span class=\"string\" title=\"$fullValue\">'$displayValue'</span>";
                 } else {
                     $args[$key] = "<span class=\"string\">'$fullValue'</span>";

tests/framework/i18n/FormatterDateTest.php

@@ -129,9 +129,9 @@ class FormatterDateTest extends TestCase
 
         // empty input
         $this->formatter->locale = 'de-DE';
-        $this->assertSame('01.01.1970 00:00:00', $this->formatter->asDatetime(''));
+        $this->assertRegExp('~01\.01\.1970,? 00:00:00~', $this->formatter->asDatetime(''));
-        $this->assertSame('01.01.1970 00:00:00', $this->formatter->asDatetime(0));
+        $this->assertRegExp('~01\.01\.1970,? 00:00:00~', $this->formatter->asDatetime(0));
-        $this->assertSame('01.01.1970 00:00:00', $this->formatter->asDatetime(false));
+        $this->assertRegExp('~01\.01\.1970,? 00:00:00~', $this->formatter->asDatetime(false));
     }
 
     public function testAsDatetime()