From 7817815dd1704d57743fc28cca172bed7a8f8036 Mon Sep 17 00:00:00 2001 From: Carsten Brandt Date: Sun, 22 Sep 2013 15:43:35 +0200 Subject: [PATCH] added more complex queries via Lua script EVAL to redis - http://redis.io/commands/eval - http://www.lua.org/ --- framework/yii/redis/ActiveQuery.php | 193 ++++++++++--- framework/yii/redis/ActiveRecord.php | 30 --- framework/yii/redis/Connection.php | 5 + framework/yii/redis/LuaScriptBuilder.php | 343 ++++++++++++++++++++++++ tests/unit/framework/redis/ActiveRecordTest.php | 29 +- 5 files changed, 518 insertions(+), 82 deletions(-) create mode 100644 framework/yii/redis/LuaScriptBuilder.php diff --git a/framework/yii/redis/ActiveQuery.php b/framework/yii/redis/ActiveQuery.php index f252898..bd597a1 100644 --- a/framework/yii/redis/ActiveQuery.php +++ b/framework/yii/redis/ActiveQuery.php @@ -66,6 +66,11 @@ class ActiveQuery extends \yii\base\Component */ public $asArray; /** + * @var array query condition. This refers to the WHERE clause in a SQL statement. + * @see where() + */ + public $where; + /** * @var integer maximum number of records to be returned. If not set or less than 0, it means no limit. */ public $limit; @@ -75,26 +80,6 @@ class ActiveQuery extends \yii\base\Component * If less than zero it means starting n elements from the end. */ public $offset; - /** - * @var array array of primary keys of the records to find. - */ - public $primaryKeys; - - /** - * List of multiple pks must be zero based - * - * @param $primaryKeys - * @return ActiveQuery - */ - public function primaryKeys($primaryKeys) { - if (is_array($primaryKeys) && isset($primaryKeys[0])) { - $this->primaryKeys = $primaryKeys; - } else { - $this->primaryKeys = array($primaryKeys); - } - - return $this; - } /** * Executes query and returns all results as an array. @@ -105,22 +90,20 @@ class ActiveQuery extends \yii\base\Component $modelClass = $this->modelClass; /** @var Connection $db */ $db = $modelClass::getDb(); - if (($primaryKeys = $this->primaryKeys) === null) { - $start = $this->offset === null ? 0 : $this->offset; - $end = $this->limit === null ? -1 : $start + $this->limit - 1; - $primaryKeys = $db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $end)); - } + + $script = $db->luaScriptBuilder->buildAll($this); + $data = $db->executeCommand('EVAL', array($script, 0)); + $rows = array(); - foreach($primaryKeys as $pk) { - $key = $modelClass::tableName() . ':a:' . $modelClass::hashPk($pk); - // get attributes - $data = $db->executeCommand('HGETALL', array($key)); + foreach($data as $dataRow) { $row = array(); - for($i=0;$icreateModels($rows); if (!empty($this->with)) { @@ -143,19 +126,16 @@ class ActiveQuery extends \yii\base\Component $modelClass = $this->modelClass; /** @var Connection $db */ $db = $modelClass::getDb(); - if (($primaryKeys = $this->primaryKeys) === null) { - $start = $this->offset === null ? 0 : $this->offset; - $primaryKeys = $db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $start)); - } - $pk = reset($primaryKeys); - $key = $modelClass::tableName() . ':a:' . $modelClass::hashPk($pk); - // get attributes - $data = $db->executeCommand('HGETALL', array($key)); + + $script = $db->luaScriptBuilder->buildOne($this); + $data = $db->executeCommand('EVAL', array($script, 0)); + if ($data === array()) { return null; } $row = array(); - for($i=0;$iasArray) { @@ -184,16 +164,29 @@ class ActiveQuery extends \yii\base\Component $modelClass = $this->modelClass; /** @var Connection $db */ $db = $modelClass::getDb(); - if ($this->offset === null && $this->limit === null) { + if ($this->offset === null && $this->limit === null && $this->where === null) { return $db->executeCommand('LLEN', array($modelClass::tableName())); } else { - $start = $this->offset === null ? 0 : $this->offset; - $end = $this->limit === null ? -1 : $start + $this->limit - 1; - return count($db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $end))); + $script = $db->luaScriptBuilder->buildCount($this); + return $db->executeCommand('EVAL', array($script, 0)); } } /** + * Returns the number of records. + * @param string $column the column to sum up + * @return integer number of records + */ + public function sum($column) + { + $modelClass = $this->modelClass; + /** @var Connection $db */ + $db = $modelClass::getDb(); + $script = $db->luaScriptBuilder->buildSum($this, $column); + return $db->executeCommand('EVAL', array($script, 0)); + } + + /** * Returns the query result as a scalar value. * The value returned will be the first column in the first row of the query results. * @return string|boolean the value of the first column in the first row of the query result. @@ -296,6 +289,118 @@ class ActiveQuery extends \yii\base\Component return $this; } + /** + * Sets the WHERE part of the query. + * + * The method requires a $condition parameter, and optionally a $params parameter + * specifying the values to be bound to the query. + * + * The $condition parameter should be either a string (e.g. 'id=1') or an array. + * If the latter, it must be in one of the following two formats: + * + * - hash format: `array('column1' => value1, 'column2' => value2, ...)` + * - operator format: `array(operator, operand1, operand2, ...)` + * + * A condition in hash format represents the following SQL expression in general: + * `column1=value1 AND column2=value2 AND ...`. In case when a value is an array, + * an `IN` expression will be generated. And if a value is null, `IS NULL` will be used + * in the generated expression. Below are some examples: + * + * - `array('type' => 1, 'status' => 2)` generates `(type = 1) AND (status = 2)`. + * - `array('id' => array(1, 2, 3), 'status' => 2)` generates `(id IN (1, 2, 3)) AND (status = 2)`. + * - `array('status' => null) generates `status IS NULL`. + * + * A condition in operator format generates the SQL expression according to the specified operator, which + * can be one of the followings: + * + * - `and`: the operands should be concatenated together using `AND`. For example, + * `array('and', 'id=1', 'id=2')` will generate `id=1 AND id=2`. If an operand is an array, + * it will be converted into a string using the rules described here. For example, + * `array('and', 'type=1', array('or', 'id=1', 'id=2'))` will generate `type=1 AND (id=1 OR id=2)`. + * The method will NOT do any quoting or escaping. + * + * - `or`: similar to the `and` operator except that the operands are concatenated using `OR`. + * + * - `between`: operand 1 should be the column name, and operand 2 and 3 should be the + * starting and ending values of the range that the column is in. + * For example, `array('between', 'id', 1, 10)` will generate `id BETWEEN 1 AND 10`. + * + * - `not between`: similar to `between` except the `BETWEEN` is replaced with `NOT BETWEEN` + * in the generated condition. + * + * - `in`: operand 1 should be a column or DB expression, and operand 2 be an array representing + * the range of the values that the column or DB expression should be in. For example, + * `array('in', 'id', array(1, 2, 3))` will generate `id IN (1, 2, 3)`. + * The method will properly quote the column name and escape values in the range. + * + * - `not in`: similar to the `in` operator except that `IN` is replaced with `NOT IN` in the generated condition. + * + * - `like`: operand 1 should be a column or DB expression, and operand 2 be a string or an array representing + * the values that the column or DB expression should be like. + * For example, `array('like', 'name', '%tester%')` will generate `name LIKE '%tester%'`. + * When the value range is given as an array, multiple `LIKE` predicates will be generated and concatenated + * using `AND`. For example, `array('like', 'name', array('%test%', '%sample%'))` will generate + * `name LIKE '%test%' AND name LIKE '%sample%'`. + * The method will properly quote the column name and escape values in the range. + * + * - `or like`: similar to the `like` operator except that `OR` is used to concatenate the `LIKE` + * predicates when operand 2 is an array. + * + * - `not like`: similar to the `like` operator except that `LIKE` is replaced with `NOT LIKE` + * in the generated condition. + * + * - `or not like`: similar to the `not like` operator except that `OR` is used to concatenate + * the `NOT LIKE` predicates. + * + * @param string|array $condition the conditions that should be put in the WHERE part. + * @return ActiveQuery the query object itself + * @see andWhere() + * @see orWhere() + */ + public function where($condition) + { + $this->where = $condition; + return $this; + } + + /** + * Adds an additional WHERE condition to the existing one. + * The new condition and the existing one will be joined using the 'AND' operator. + * @param string|array $condition the new WHERE condition. Please refer to [[where()]] + * on how to specify this parameter. + * @return ActiveQuery the query object itself + * @see where() + * @see orWhere() + */ + public function andWhere($condition) + { + if ($this->where === null) { + $this->where = $condition; + } else { + $this->where = array('and', $this->where, $condition); + } + return $this; + } + + /** + * Adds an additional WHERE condition to the existing one. + * The new condition and the existing one will be joined using the 'OR' operator. + * @param string|array $condition the new WHERE condition. Please refer to [[where()]] + * on how to specify this parameter. + * @return ActiveQuery the query object itself + * @see where() + * @see andWhere() + */ + public function orWhere($condition) + { + if ($this->where === null) { + $this->where = $condition; + } else { + $this->where = array('or', $this->where, $condition); + } + return $this; + } + // TODO: refactor, it is duplicated from yii/db/ActiveQuery private function createModels($rows) { diff --git a/framework/yii/redis/ActiveRecord.php b/framework/yii/redis/ActiveRecord.php index 44cd5d7..0a1f7bd 100644 --- a/framework/yii/redis/ActiveRecord.php +++ b/framework/yii/redis/ActiveRecord.php @@ -38,36 +38,6 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord return \Yii::$app->redis; } - /** - * Creates an [[ActiveQuery]] instance for query purpose. - * - * @include @yii/db/ActiveRecord-find.md - * - * @param mixed $q the query parameter. This can be one of the followings: - * - * - a scalar value (integer or string): query by a single primary key value and return the - * corresponding record. - * - an array of name-value pairs: query by a set of column values and return a single record matching all of them. - * - null: return a new [[ActiveQuery]] object for further query purpose. - * - * @return ActiveQuery|ActiveRecord|null When `$q` is null, a new [[ActiveQuery]] instance - * is returned; when `$q` is a scalar or an array, an ActiveRecord object matching it will be - * returned (null will be returned if there is no matching). - * @see createQuery() - */ - public static function find($q = null) // TODO optimize API - { - $query = static::createQuery(); - if (is_array($q)) { - return $query->primaryKeys($q)->one(); - } elseif ($q !== null) { - // query by primary key - $primaryKey = static::primaryKey(); - return $query->primaryKeys(array($primaryKey[0] => $q))->one(); - } - return $query; - } - public static function hashPk($pk) { return is_array($pk) ? implode('-', $pk) : $pk; // TODO escape PK glue diff --git a/framework/yii/redis/Connection.php b/framework/yii/redis/Connection.php index 848b408..cc6c253 100644 --- a/framework/yii/redis/Connection.php +++ b/framework/yii/redis/Connection.php @@ -333,6 +333,11 @@ class Connection extends Component } } + public function getLuaScriptBuilder() + { + return new LuaScriptBuilder(); + } + /** * * @param string $name diff --git a/framework/yii/redis/LuaScriptBuilder.php b/framework/yii/redis/LuaScriptBuilder.php new file mode 100644 index 0000000..f9b6cf6 --- /dev/null +++ b/framework/yii/redis/LuaScriptBuilder.php @@ -0,0 +1,343 @@ + + * @since 2.0 + */ +class LuaScriptBuilder extends \yii\base\Object +{ + public function buildAll($query) + { + $modelClass = $query->modelClass; + $key = $modelClass::tableName(); + return $this->build($query, "n=n+1 pks[n] = redis.call('HGETALL','$key:a:' .. pk)", 'pks'); // TODO quote + } + + public function buildOne($query) + { + $modelClass = $query->modelClass; + $key = $modelClass::tableName(); + return $this->build($query, "do return redis.call('HGETALL','$key:a:' .. pk) end", 'pks'); // TODO quote + } + + public function buildCount($query) + { + return $this->build($query, 'n=n+1', 'n'); + } + + public function buildSum($query, $field) + { + $modelClass = $query->modelClass; + $key = $modelClass::tableName(); + return $this->build($query, "n=n+redis.call('HGET','$key:a:' .. pk,'$field')", 'n'); // TODO quote + } + + /** + * @param ActiveQuery $query + */ + public function build($query, $buildResult, $return) + { + $columns = array(); + if ($query->where !== null) { + $condition = $this->buildCondition($query->where, $columns); + } else { + $condition = 'true'; + } + + $start = $query->offset === null ? 0 : $query->offset; + $limitCondition = 'i>' . $start . ($query->limit === null ? '' : ' and i<=' . ($start + $query->limit)); + + $modelClass = $query->modelClass; + $key = $modelClass::tableName(); + $loadColumnValues = ''; + foreach($columns as $column) { + $loadColumnValues .= "local $column=redis.call('HGET','$key:a:' .. pk, '$column')\n"; // TODO properly hash pk + } + + return <<buildColumns($columns); + } + + /** + * @param string|array $condition + * @param array $params the binding parameters to be populated + * @return string the HAVING clause built from [[query]]. + */ + public function buildHaving($condition, &$params) + { + $having = $this->buildCondition($condition, $params); + return $having === '' ? '' : 'HAVING ' . $having; + } + + /** + * @param array $columns + * @return string the ORDER BY clause built from [[query]]. + */ + public function buildOrderBy($columns) + { + if (empty($columns)) { + return ''; + } + $orders = array(); + foreach ($columns as $name => $direction) { + if (is_object($direction)) { + $orders[] = (string)$direction; + } else { + $orders[] = $this->db->quoteColumnName($name) . ($direction === Query::SORT_DESC ? ' DESC' : ''); + } + } + + return 'ORDER BY ' . implode(', ', $orders); + } + + /** + * Parses the condition specification and generates the corresponding SQL expression. + * @param string|array $condition the condition specification. Please refer to [[Query::where()]] + * on how to specify a condition. + * @param array $params the binding parameters to be populated + * @return string the generated SQL expression + * @throws \yii\db\Exception if the condition is in bad format + */ + public function buildCondition($condition, &$columns) + { + static $builders = array( + 'and' => 'buildAndCondition', + 'or' => 'buildAndCondition', + 'between' => 'buildBetweenCondition', + 'not between' => 'buildBetweenCondition', + 'in' => 'buildInCondition', + 'not in' => 'buildInCondition', + 'like' => 'buildLikeCondition', + 'not like' => 'buildLikeCondition', + 'or like' => 'buildLikeCondition', + 'or not like' => 'buildLikeCondition', + ); + + if (!is_array($condition)) { + throw new NotSupportedException('Where must be an array.'); + } + if (isset($condition[0])) { // operator format: operator, operand 1, operand 2, ... + $operator = strtolower($condition[0]); + if (isset($builders[$operator])) { + $method = $builders[$operator]; + array_shift($condition); + return $this->$method($operator, $condition, $columns); + } else { + throw new Exception('Found unknown operator in query: ' . $operator); + } + } else { // hash format: 'column1' => 'value1', 'column2' => 'value2', ... + return $this->buildHashCondition($condition, $columns); + } + } + + private function buildHashCondition($condition, &$columns) + { + $parts = array(); + foreach ($condition as $column => $value) { + // TODO replace special chars and keywords in column name + $columns[$column] = $column; + if (is_array($value)) { // IN condition + $parts[] = $this->buildInCondition('IN', array($column, $value), $columns); + } else { + if ($value === null) { + $parts[] = $column.'==nil'; + } elseif ($value instanceof Expression) { + $parts[] = "$column==" . $value->expression; + } else { + $value = $this->quoteValue($value); + $parts[] = "$column==$value"; + } + } + } + return count($parts) === 1 ? $parts[0] : '(' . implode(') and (', $parts) . ')'; + } + + private function buildAndCondition($operator, $operands, &$columns) + { + $parts = array(); + foreach ($operands as $operand) { + if (is_array($operand)) { + $operand = $this->buildCondition($operand, $columns); + } + if ($operand !== '') { + $parts[] = $operand; + } + } + if (!empty($parts)) { + return '(' . implode(") $operator (", $parts) . ')'; + } else { + return ''; + } + } + + private function buildBetweenCondition($operator, $operands, &$columns) + { + if (!isset($operands[0], $operands[1], $operands[2])) { + throw new Exception("Operator '$operator' requires three operands."); + } + + list($column, $value1, $value2) = $operands; + + // TODO replace special chars and keywords in column name + $value1 = $this->quoteValue($value1); + $value2 = $this->quoteValue($value2); + $columns[$column] = $column; + return "$column > $value1 and $column < $value2"; + } + + private function buildInCondition($operator, $operands, &$columns) + { + // TODO adjust implementation to respect NOT IN operator + if (!isset($operands[0], $operands[1])) { + throw new Exception("Operator '$operator' requires two operands."); + } + + list($column, $values) = $operands; + + $values = (array)$values; + + if (empty($values) || $column === array()) { + return $operator === 'IN' ? '0==1' : ''; + } + + if (count($column) > 1) { + return $this->buildCompositeInCondition($operator, $column, $values, $columns); + } elseif (is_array($column)) { + $column = reset($column); + } + $parts = array(); + foreach ($values as $i => $value) { + if (is_array($value)) { + $value = isset($value[$column]) ? $value[$column] : null; + } + // TODO replace special chars and keywords in column name + if ($value === null) { + $parts[] = 'type('.$column.')=="nil"'; + } elseif ($value instanceof Expression) { + $parts[] = "$column==" . $value->expression; + } else { + $value = $this->quoteValue($value); + $parts[] = "$column==$value"; + } + } + if (count($parts) > 1) { + return "(" . implode(' or ', $parts) . ')'; + } else { + $operator = $operator === 'IN' ? '' : '!'; + return "$operator({$values[0]})"; + } + } + + protected function buildCompositeInCondition($operator, $columns, $values, &$params) + { + throw new NotSupportedException('composie IN is not yet supported.'); + // TODO implement correclty + $vss = array(); + foreach ($values as $value) { + $vs = array(); + foreach ($columns as $column) { + if (isset($value[$column])) { + $phName = self::PARAM_PREFIX . count($params); + $params[$phName] = $value[$column]; + $vs[] = $phName; + } else { + $vs[] = 'NULL'; + } + } + $vss[] = '(' . implode(', ', $vs) . ')'; + } + foreach ($columns as $i => $column) { + if (strpos($column, '(') === false) { + $columns[$i] = $this->db->quoteColumnName($column); + } + } + return '(' . implode(', ', $columns) . ") $operator (" . implode(', ', $vss) . ')'; + } + + private function buildLikeCondition($operator, $operands, &$params) + { + throw new NotSupportedException('LIKE is not yet supported.'); + // TODO implement correclty + if (!isset($operands[0], $operands[1])) { + throw new Exception("Operator '$operator' requires two operands."); + } + + list($column, $values) = $operands; + + $values = (array)$values; + + if (empty($values)) { + return $operator === 'LIKE' || $operator === 'OR LIKE' ? '0=1' : ''; + } + + if ($operator === 'LIKE' || $operator === 'NOT LIKE') { + $andor = ' AND '; + } else { + $andor = ' OR '; + $operator = $operator === 'OR LIKE' ? 'LIKE' : 'NOT LIKE'; + } + + if (strpos($column, '(') === false) { + $column = $this->db->quoteColumnName($column); + } + + $parts = array(); + foreach ($values as $value) { + $phName = self::PARAM_PREFIX . count($params); + $params[$phName] = $value; + $parts[] = "$column $operator $phName"; + } + + return implode($andor, $parts); + } +} diff --git a/tests/unit/framework/redis/ActiveRecordTest.php b/tests/unit/framework/redis/ActiveRecordTest.php index 53ea548..a3a5559 100644 --- a/tests/unit/framework/redis/ActiveRecordTest.php +++ b/tests/unit/framework/redis/ActiveRecordTest.php @@ -118,7 +118,7 @@ class ActiveRecordTest extends RedisTestCase $this->assertNull($customer); // query scalar - $customerName = Customer::find()->primaryKeys(2)->scalar('name'); + $customerName = Customer::find()->where(array('id' => 2))->scalar('name'); $this->assertEquals('user2', $customerName); // find by column values @@ -129,13 +129,12 @@ class ActiveRecordTest extends RedisTestCase $this->assertNull($customer); // find by attributes -/* $customer = Customer::find()->where(array('name' => 'user2'))->one(); + $customer = Customer::find()->where(array('name' => 'user2'))->one(); $this->assertTrue($customer instanceof Customer); - $this->assertEquals(2, $customer->id);*/ + $this->assertEquals(2, $customer->id); // find count, sum, average, min, max, scalar -/* $this->assertEquals(2, Customer::find()->where('id=1 OR id=2')->count()); - $this->assertEquals(6, Customer::find()->sum('id')); +/* $this->assertEquals(6, Customer::find()->sum('id')); $this->assertEquals(2, Customer::find()->average('id')); $this->assertEquals(1, Customer::find()->min('id')); $this->assertEquals(3, Customer::find()->max('id')); @@ -145,7 +144,7 @@ class ActiveRecordTest extends RedisTestCase // $this->assertEquals(2, Customer::find()->active()->count()); // asArray - $customer = Customer::find()->primaryKeys(array(2))->asArray()->one(); + $customer = Customer::find()->where(array('id' => 2))->asArray()->one(); $this->assertEquals(array( 'id' => '2', 'email' => 'user2@example.com', @@ -214,10 +213,24 @@ class ActiveRecordTest extends RedisTestCase } + public function testFindComplexCondition() + { + $this->assertEquals(2, Customer::find()->where(array('OR', array('id' => 1), array('id' => 2)))->count()); + $this->assertEquals(2, count(Customer::find()->where(array('OR', array('id' => 1), array('id' => 2)))->all())); + + // TODO more conditions + } + + public function testSum() + { + $this->assertEquals(6, OrderItem::find()->count()); + $this->assertEquals(7, OrderItem::find()->sum('quantity')); + } + public function testExists() { - $this->assertTrue(Customer::find()->primaryKeys(2)->exists()); - $this->assertFalse(Customer::find()->primaryKeys(5)->exists()); + $this->assertTrue(Customer::find()->where(array('id' => 2))->exists()); + $this->assertFalse(Customer::find()->where(array('id' => 5))->exists()); } // public function testFindLazy()