Browse Source

added more complex queries via Lua script EVAL to redis

- http://redis.io/commands/eval
- http://www.lua.org/
tags/2.0.0-beta
Carsten Brandt 11 years ago
parent
commit
7817815dd1
  1. 193
      framework/yii/redis/ActiveQuery.php
  2. 30
      framework/yii/redis/ActiveRecord.php
  3. 5
      framework/yii/redis/Connection.php
  4. 343
      framework/yii/redis/LuaScriptBuilder.php
  5. 29
      tests/unit/framework/redis/ActiveRecordTest.php

193
framework/yii/redis/ActiveQuery.php

@ -66,6 +66,11 @@ class ActiveQuery extends \yii\base\Component
*/ */
public $asArray; public $asArray;
/** /**
* @var array query condition. This refers to the WHERE clause in a SQL statement.
* @see where()
*/
public $where;
/**
* @var integer maximum number of records to be returned. If not set or less than 0, it means no limit. * @var integer maximum number of records to be returned. If not set or less than 0, it means no limit.
*/ */
public $limit; public $limit;
@ -75,26 +80,6 @@ class ActiveQuery extends \yii\base\Component
* If less than zero it means starting n elements from the end. * If less than zero it means starting n elements from the end.
*/ */
public $offset; public $offset;
/**
* @var array array of primary keys of the records to find.
*/
public $primaryKeys;
/**
* List of multiple pks must be zero based
*
* @param $primaryKeys
* @return ActiveQuery
*/
public function primaryKeys($primaryKeys) {
if (is_array($primaryKeys) && isset($primaryKeys[0])) {
$this->primaryKeys = $primaryKeys;
} else {
$this->primaryKeys = array($primaryKeys);
}
return $this;
}
/** /**
* Executes query and returns all results as an array. * Executes query and returns all results as an array.
@ -105,22 +90,20 @@ class ActiveQuery extends \yii\base\Component
$modelClass = $this->modelClass; $modelClass = $this->modelClass;
/** @var Connection $db */ /** @var Connection $db */
$db = $modelClass::getDb(); $db = $modelClass::getDb();
if (($primaryKeys = $this->primaryKeys) === null) {
$start = $this->offset === null ? 0 : $this->offset; $script = $db->luaScriptBuilder->buildAll($this);
$end = $this->limit === null ? -1 : $start + $this->limit - 1; $data = $db->executeCommand('EVAL', array($script, 0));
$primaryKeys = $db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $end));
}
$rows = array(); $rows = array();
foreach($primaryKeys as $pk) { foreach($data as $dataRow) {
$key = $modelClass::tableName() . ':a:' . $modelClass::hashPk($pk);
// get attributes
$data = $db->executeCommand('HGETALL', array($key));
$row = array(); $row = array();
for($i=0;$i<count($data);) { $c = count($dataRow);
$row[$data[$i++]] = $data[$i++]; for($i = 0; $i < $c; ) {
$row[$dataRow[$i++]] = $dataRow[$i++];
} }
$rows[] = $row; $rows[] = $row;
} }
if ($rows !== array()) { if ($rows !== array()) {
$models = $this->createModels($rows); $models = $this->createModels($rows);
if (!empty($this->with)) { if (!empty($this->with)) {
@ -143,19 +126,16 @@ class ActiveQuery extends \yii\base\Component
$modelClass = $this->modelClass; $modelClass = $this->modelClass;
/** @var Connection $db */ /** @var Connection $db */
$db = $modelClass::getDb(); $db = $modelClass::getDb();
if (($primaryKeys = $this->primaryKeys) === null) {
$start = $this->offset === null ? 0 : $this->offset; $script = $db->luaScriptBuilder->buildOne($this);
$primaryKeys = $db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $start)); $data = $db->executeCommand('EVAL', array($script, 0));
}
$pk = reset($primaryKeys);
$key = $modelClass::tableName() . ':a:' . $modelClass::hashPk($pk);
// get attributes
$data = $db->executeCommand('HGETALL', array($key));
if ($data === array()) { if ($data === array()) {
return null; return null;
} }
$row = array(); $row = array();
for($i=0;$i<count($data);) { $c = count($data);
for($i = 0; $i < $c; ) {
$row[$data[$i++]] = $data[$i++]; $row[$data[$i++]] = $data[$i++];
} }
if (!$this->asArray) { if (!$this->asArray) {
@ -184,13 +164,26 @@ class ActiveQuery extends \yii\base\Component
$modelClass = $this->modelClass; $modelClass = $this->modelClass;
/** @var Connection $db */ /** @var Connection $db */
$db = $modelClass::getDb(); $db = $modelClass::getDb();
if ($this->offset === null && $this->limit === null) { if ($this->offset === null && $this->limit === null && $this->where === null) {
return $db->executeCommand('LLEN', array($modelClass::tableName())); return $db->executeCommand('LLEN', array($modelClass::tableName()));
} else { } else {
$start = $this->offset === null ? 0 : $this->offset; $script = $db->luaScriptBuilder->buildCount($this);
$end = $this->limit === null ? -1 : $start + $this->limit - 1; return $db->executeCommand('EVAL', array($script, 0));
return count($db->executeCommand('LRANGE', array($modelClass::tableName(), $start, $end))); }
} }
/**
* Returns the number of records.
* @param string $column the column to sum up
* @return integer number of records
*/
public function sum($column)
{
$modelClass = $this->modelClass;
/** @var Connection $db */
$db = $modelClass::getDb();
$script = $db->luaScriptBuilder->buildSum($this, $column);
return $db->executeCommand('EVAL', array($script, 0));
} }
/** /**
@ -296,6 +289,118 @@ class ActiveQuery extends \yii\base\Component
return $this; return $this;
} }
/**
* Sets the WHERE part of the query.
*
* The method requires a $condition parameter, and optionally a $params parameter
* specifying the values to be bound to the query.
*
* The $condition parameter should be either a string (e.g. 'id=1') or an array.
* If the latter, it must be in one of the following two formats:
*
* - hash format: `array('column1' => value1, 'column2' => value2, ...)`
* - operator format: `array(operator, operand1, operand2, ...)`
*
* A condition in hash format represents the following SQL expression in general:
* `column1=value1 AND column2=value2 AND ...`. In case when a value is an array,
* an `IN` expression will be generated. And if a value is null, `IS NULL` will be used
* in the generated expression. Below are some examples:
*
* - `array('type' => 1, 'status' => 2)` generates `(type = 1) AND (status = 2)`.
* - `array('id' => array(1, 2, 3), 'status' => 2)` generates `(id IN (1, 2, 3)) AND (status = 2)`.
* - `array('status' => null) generates `status IS NULL`.
*
* A condition in operator format generates the SQL expression according to the specified operator, which
* can be one of the followings:
*
* - `and`: the operands should be concatenated together using `AND`. For example,
* `array('and', 'id=1', 'id=2')` will generate `id=1 AND id=2`. If an operand is an array,
* it will be converted into a string using the rules described here. For example,
* `array('and', 'type=1', array('or', 'id=1', 'id=2'))` will generate `type=1 AND (id=1 OR id=2)`.
* The method will NOT do any quoting or escaping.
*
* - `or`: similar to the `and` operator except that the operands are concatenated using `OR`.
*
* - `between`: operand 1 should be the column name, and operand 2 and 3 should be the
* starting and ending values of the range that the column is in.
* For example, `array('between', 'id', 1, 10)` will generate `id BETWEEN 1 AND 10`.
*
* - `not between`: similar to `between` except the `BETWEEN` is replaced with `NOT BETWEEN`
* in the generated condition.
*
* - `in`: operand 1 should be a column or DB expression, and operand 2 be an array representing
* the range of the values that the column or DB expression should be in. For example,
* `array('in', 'id', array(1, 2, 3))` will generate `id IN (1, 2, 3)`.
* The method will properly quote the column name and escape values in the range.
*
* - `not in`: similar to the `in` operator except that `IN` is replaced with `NOT IN` in the generated condition.
*
* - `like`: operand 1 should be a column or DB expression, and operand 2 be a string or an array representing
* the values that the column or DB expression should be like.
* For example, `array('like', 'name', '%tester%')` will generate `name LIKE '%tester%'`.
* When the value range is given as an array, multiple `LIKE` predicates will be generated and concatenated
* using `AND`. For example, `array('like', 'name', array('%test%', '%sample%'))` will generate
* `name LIKE '%test%' AND name LIKE '%sample%'`.
* The method will properly quote the column name and escape values in the range.
*
* - `or like`: similar to the `like` operator except that `OR` is used to concatenate the `LIKE`
* predicates when operand 2 is an array.
*
* - `not like`: similar to the `like` operator except that `LIKE` is replaced with `NOT LIKE`
* in the generated condition.
*
* - `or not like`: similar to the `not like` operator except that `OR` is used to concatenate
* the `NOT LIKE` predicates.
*
* @param string|array $condition the conditions that should be put in the WHERE part.
* @return ActiveQuery the query object itself
* @see andWhere()
* @see orWhere()
*/
public function where($condition)
{
$this->where = $condition;
return $this;
}
/**
* Adds an additional WHERE condition to the existing one.
* The new condition and the existing one will be joined using the 'AND' operator.
* @param string|array $condition the new WHERE condition. Please refer to [[where()]]
* on how to specify this parameter.
* @return ActiveQuery the query object itself
* @see where()
* @see orWhere()
*/
public function andWhere($condition)
{
if ($this->where === null) {
$this->where = $condition;
} else {
$this->where = array('and', $this->where, $condition);
}
return $this;
}
/**
* Adds an additional WHERE condition to the existing one.
* The new condition and the existing one will be joined using the 'OR' operator.
* @param string|array $condition the new WHERE condition. Please refer to [[where()]]
* on how to specify this parameter.
* @return ActiveQuery the query object itself
* @see where()
* @see andWhere()
*/
public function orWhere($condition)
{
if ($this->where === null) {
$this->where = $condition;
} else {
$this->where = array('or', $this->where, $condition);
}
return $this;
}
// TODO: refactor, it is duplicated from yii/db/ActiveQuery // TODO: refactor, it is duplicated from yii/db/ActiveQuery
private function createModels($rows) private function createModels($rows)
{ {

30
framework/yii/redis/ActiveRecord.php

@ -38,36 +38,6 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord
return \Yii::$app->redis; return \Yii::$app->redis;
} }
/**
* Creates an [[ActiveQuery]] instance for query purpose.
*
* @include @yii/db/ActiveRecord-find.md
*
* @param mixed $q the query parameter. This can be one of the followings:
*
* - a scalar value (integer or string): query by a single primary key value and return the
* corresponding record.
* - an array of name-value pairs: query by a set of column values and return a single record matching all of them.
* - null: return a new [[ActiveQuery]] object for further query purpose.
*
* @return ActiveQuery|ActiveRecord|null When `$q` is null, a new [[ActiveQuery]] instance
* is returned; when `$q` is a scalar or an array, an ActiveRecord object matching it will be
* returned (null will be returned if there is no matching).
* @see createQuery()
*/
public static function find($q = null) // TODO optimize API
{
$query = static::createQuery();
if (is_array($q)) {
return $query->primaryKeys($q)->one();
} elseif ($q !== null) {
// query by primary key
$primaryKey = static::primaryKey();
return $query->primaryKeys(array($primaryKey[0] => $q))->one();
}
return $query;
}
public static function hashPk($pk) public static function hashPk($pk)
{ {
return is_array($pk) ? implode('-', $pk) : $pk; // TODO escape PK glue return is_array($pk) ? implode('-', $pk) : $pk; // TODO escape PK glue

5
framework/yii/redis/Connection.php

@ -333,6 +333,11 @@ class Connection extends Component
} }
} }
public function getLuaScriptBuilder()
{
return new LuaScriptBuilder();
}
/** /**
* *
* @param string $name * @param string $name

343
framework/yii/redis/LuaScriptBuilder.php

@ -0,0 +1,343 @@
<?php
/**
* @link http://www.yiiframework.com/
* @copyright Copyright (c) 2008 Yii Software LLC
* @license http://www.yiiframework.com/license/
*/
namespace yii\redis;
use yii\base\NotSupportedException;
/**
* LuaScriptBuilder builds lua scripts used for retrieving data from redis.
*
* @author Carsten Brandt <mail@cebe.cc>
* @since 2.0
*/
class LuaScriptBuilder extends \yii\base\Object
{
public function buildAll($query)
{
$modelClass = $query->modelClass;
$key = $modelClass::tableName();
return $this->build($query, "n=n+1 pks[n] = redis.call('HGETALL','$key:a:' .. pk)", 'pks'); // TODO quote
}
public function buildOne($query)
{
$modelClass = $query->modelClass;
$key = $modelClass::tableName();
return $this->build($query, "do return redis.call('HGETALL','$key:a:' .. pk) end", 'pks'); // TODO quote
}
public function buildCount($query)
{
return $this->build($query, 'n=n+1', 'n');
}
public function buildSum($query, $field)
{
$modelClass = $query->modelClass;
$key = $modelClass::tableName();
return $this->build($query, "n=n+redis.call('HGET','$key:a:' .. pk,'$field')", 'n'); // TODO quote
}
/**
* @param ActiveQuery $query
*/
public function build($query, $buildResult, $return)
{
$columns = array();
if ($query->where !== null) {
$condition = $this->buildCondition($query->where, $columns);
} else {
$condition = 'true';
}
$start = $query->offset === null ? 0 : $query->offset;
$limitCondition = 'i>' . $start . ($query->limit === null ? '' : ' and i<=' . ($start + $query->limit));
$modelClass = $query->modelClass;
$key = $modelClass::tableName();
$loadColumnValues = '';
foreach($columns as $column) {
$loadColumnValues .= "local $column=redis.call('HGET','$key:a:' .. pk, '$column')\n"; // TODO properly hash pk
}
return <<<EOF
local allpks=redis.call('LRANGE','$key',0,-1)
local pks={}
local n=0
local i=0
for k,pk in ipairs(allpks) do
$loadColumnValues
if $condition then
i=i+1
if $limitCondition then
$buildResult
end
end
end
return $return
EOF;
}
/**
* Quotes a string value for use in a query.
* Note that if the parameter is not a string, it will be returned without change.
* @param string $str string to be quoted
* @return string the properly quoted string
* @see http://www.php.net/manual/en/function.PDO-quote.php
*/
public function quoteValue($str)
{
if (!is_string($str) && !is_int($str)) {
return $str;
}
return "'" . addcslashes(str_replace("'", "\\'", $str), "\000\n\r\\\032") . "'";
}
/**
* @param array $columns
* @return string the GROUP BY clause
*/
public function buildGroupBy($columns)
{
return empty($columns) ? '' : 'GROUP BY ' . $this->buildColumns($columns);
}
/**
* @param string|array $condition
* @param array $params the binding parameters to be populated
* @return string the HAVING clause built from [[query]].
*/
public function buildHaving($condition, &$params)
{
$having = $this->buildCondition($condition, $params);
return $having === '' ? '' : 'HAVING ' . $having;
}
/**
* @param array $columns
* @return string the ORDER BY clause built from [[query]].
*/
public function buildOrderBy($columns)
{
if (empty($columns)) {
return '';
}
$orders = array();
foreach ($columns as $name => $direction) {
if (is_object($direction)) {
$orders[] = (string)$direction;
} else {
$orders[] = $this->db->quoteColumnName($name) . ($direction === Query::SORT_DESC ? ' DESC' : '');
}
}
return 'ORDER BY ' . implode(', ', $orders);
}
/**
* Parses the condition specification and generates the corresponding SQL expression.
* @param string|array $condition the condition specification. Please refer to [[Query::where()]]
* on how to specify a condition.
* @param array $params the binding parameters to be populated
* @return string the generated SQL expression
* @throws \yii\db\Exception if the condition is in bad format
*/
public function buildCondition($condition, &$columns)
{
static $builders = array(
'and' => 'buildAndCondition',
'or' => 'buildAndCondition',
'between' => 'buildBetweenCondition',
'not between' => 'buildBetweenCondition',
'in' => 'buildInCondition',
'not in' => 'buildInCondition',
'like' => 'buildLikeCondition',
'not like' => 'buildLikeCondition',
'or like' => 'buildLikeCondition',
'or not like' => 'buildLikeCondition',
);
if (!is_array($condition)) {
throw new NotSupportedException('Where must be an array.');
}
if (isset($condition[0])) { // operator format: operator, operand 1, operand 2, ...
$operator = strtolower($condition[0]);
if (isset($builders[$operator])) {
$method = $builders[$operator];
array_shift($condition);
return $this->$method($operator, $condition, $columns);
} else {
throw new Exception('Found unknown operator in query: ' . $operator);
}
} else { // hash format: 'column1' => 'value1', 'column2' => 'value2', ...
return $this->buildHashCondition($condition, $columns);
}
}
private function buildHashCondition($condition, &$columns)
{
$parts = array();
foreach ($condition as $column => $value) {
// TODO replace special chars and keywords in column name
$columns[$column] = $column;
if (is_array($value)) { // IN condition
$parts[] = $this->buildInCondition('IN', array($column, $value), $columns);
} else {
if ($value === null) {
$parts[] = $column.'==nil';
} elseif ($value instanceof Expression) {
$parts[] = "$column==" . $value->expression;
} else {
$value = $this->quoteValue($value);
$parts[] = "$column==$value";
}
}
}
return count($parts) === 1 ? $parts[0] : '(' . implode(') and (', $parts) . ')';
}
private function buildAndCondition($operator, $operands, &$columns)
{
$parts = array();
foreach ($operands as $operand) {
if (is_array($operand)) {
$operand = $this->buildCondition($operand, $columns);
}
if ($operand !== '') {
$parts[] = $operand;
}
}
if (!empty($parts)) {
return '(' . implode(") $operator (", $parts) . ')';
} else {
return '';
}
}
private function buildBetweenCondition($operator, $operands, &$columns)
{
if (!isset($operands[0], $operands[1], $operands[2])) {
throw new Exception("Operator '$operator' requires three operands.");
}
list($column, $value1, $value2) = $operands;
// TODO replace special chars and keywords in column name
$value1 = $this->quoteValue($value1);
$value2 = $this->quoteValue($value2);
$columns[$column] = $column;
return "$column > $value1 and $column < $value2";
}
private function buildInCondition($operator, $operands, &$columns)
{
// TODO adjust implementation to respect NOT IN operator
if (!isset($operands[0], $operands[1])) {
throw new Exception("Operator '$operator' requires two operands.");
}
list($column, $values) = $operands;
$values = (array)$values;
if (empty($values) || $column === array()) {
return $operator === 'IN' ? '0==1' : '';
}
if (count($column) > 1) {
return $this->buildCompositeInCondition($operator, $column, $values, $columns);
} elseif (is_array($column)) {
$column = reset($column);
}
$parts = array();
foreach ($values as $i => $value) {
if (is_array($value)) {
$value = isset($value[$column]) ? $value[$column] : null;
}
// TODO replace special chars and keywords in column name
if ($value === null) {
$parts[] = 'type('.$column.')=="nil"';
} elseif ($value instanceof Expression) {
$parts[] = "$column==" . $value->expression;
} else {
$value = $this->quoteValue($value);
$parts[] = "$column==$value";
}
}
if (count($parts) > 1) {
return "(" . implode(' or ', $parts) . ')';
} else {
$operator = $operator === 'IN' ? '' : '!';
return "$operator({$values[0]})";
}
}
protected function buildCompositeInCondition($operator, $columns, $values, &$params)
{
throw new NotSupportedException('composie IN is not yet supported.');
// TODO implement correclty
$vss = array();
foreach ($values as $value) {
$vs = array();
foreach ($columns as $column) {
if (isset($value[$column])) {
$phName = self::PARAM_PREFIX . count($params);
$params[$phName] = $value[$column];
$vs[] = $phName;
} else {
$vs[] = 'NULL';
}
}
$vss[] = '(' . implode(', ', $vs) . ')';
}
foreach ($columns as $i => $column) {
if (strpos($column, '(') === false) {
$columns[$i] = $this->db->quoteColumnName($column);
}
}
return '(' . implode(', ', $columns) . ") $operator (" . implode(', ', $vss) . ')';
}
private function buildLikeCondition($operator, $operands, &$params)
{
throw new NotSupportedException('LIKE is not yet supported.');
// TODO implement correclty
if (!isset($operands[0], $operands[1])) {
throw new Exception("Operator '$operator' requires two operands.");
}
list($column, $values) = $operands;
$values = (array)$values;
if (empty($values)) {
return $operator === 'LIKE' || $operator === 'OR LIKE' ? '0=1' : '';
}
if ($operator === 'LIKE' || $operator === 'NOT LIKE') {
$andor = ' AND ';
} else {
$andor = ' OR ';
$operator = $operator === 'OR LIKE' ? 'LIKE' : 'NOT LIKE';
}
if (strpos($column, '(') === false) {
$column = $this->db->quoteColumnName($column);
}
$parts = array();
foreach ($values as $value) {
$phName = self::PARAM_PREFIX . count($params);
$params[$phName] = $value;
$parts[] = "$column $operator $phName";
}
return implode($andor, $parts);
}
}

29
tests/unit/framework/redis/ActiveRecordTest.php

@ -118,7 +118,7 @@ class ActiveRecordTest extends RedisTestCase
$this->assertNull($customer); $this->assertNull($customer);
// query scalar // query scalar
$customerName = Customer::find()->primaryKeys(2)->scalar('name'); $customerName = Customer::find()->where(array('id' => 2))->scalar('name');
$this->assertEquals('user2', $customerName); $this->assertEquals('user2', $customerName);
// find by column values // find by column values
@ -129,13 +129,12 @@ class ActiveRecordTest extends RedisTestCase
$this->assertNull($customer); $this->assertNull($customer);
// find by attributes // find by attributes
/* $customer = Customer::find()->where(array('name' => 'user2'))->one(); $customer = Customer::find()->where(array('name' => 'user2'))->one();
$this->assertTrue($customer instanceof Customer); $this->assertTrue($customer instanceof Customer);
$this->assertEquals(2, $customer->id);*/ $this->assertEquals(2, $customer->id);
// find count, sum, average, min, max, scalar // find count, sum, average, min, max, scalar
/* $this->assertEquals(2, Customer::find()->where('id=1 OR id=2')->count()); /* $this->assertEquals(6, Customer::find()->sum('id'));
$this->assertEquals(6, Customer::find()->sum('id'));
$this->assertEquals(2, Customer::find()->average('id')); $this->assertEquals(2, Customer::find()->average('id'));
$this->assertEquals(1, Customer::find()->min('id')); $this->assertEquals(1, Customer::find()->min('id'));
$this->assertEquals(3, Customer::find()->max('id')); $this->assertEquals(3, Customer::find()->max('id'));
@ -145,7 +144,7 @@ class ActiveRecordTest extends RedisTestCase
// $this->assertEquals(2, Customer::find()->active()->count()); // $this->assertEquals(2, Customer::find()->active()->count());
// asArray // asArray
$customer = Customer::find()->primaryKeys(array(2))->asArray()->one(); $customer = Customer::find()->where(array('id' => 2))->asArray()->one();
$this->assertEquals(array( $this->assertEquals(array(
'id' => '2', 'id' => '2',
'email' => 'user2@example.com', 'email' => 'user2@example.com',
@ -214,10 +213,24 @@ class ActiveRecordTest extends RedisTestCase
} }
public function testFindComplexCondition()
{
$this->assertEquals(2, Customer::find()->where(array('OR', array('id' => 1), array('id' => 2)))->count());
$this->assertEquals(2, count(Customer::find()->where(array('OR', array('id' => 1), array('id' => 2)))->all()));
// TODO more conditions
}
public function testSum()
{
$this->assertEquals(6, OrderItem::find()->count());
$this->assertEquals(7, OrderItem::find()->sum('quantity'));
}
public function testExists() public function testExists()
{ {
$this->assertTrue(Customer::find()->primaryKeys(2)->exists()); $this->assertTrue(Customer::find()->where(array('id' => 2))->exists());
$this->assertFalse(Customer::find()->primaryKeys(5)->exists()); $this->assertFalse(Customer::find()->where(array('id' => 5))->exists());
} }
// public function testFindLazy() // public function testFindLazy()

Loading…
Cancel
Save