Configure Redis 7 Lua scripting and modules for advanced data processing

Install Redis 7 with module support

Start by installing Redis 7 and development tools needed for module compilation.

sudo apt update
sudo apt install -y redis-server redis-tools build-essential git cmake python3-dev

sudo dnf update -y
sudo dnf install -y redis redis-devel gcc gcc-c++ make git cmake python3-devel
sudo dnf groupinstall -y "Development Tools"

Configure Redis for Lua scripting

Enable Lua scripting and configure memory limits for script execution in the Redis configuration.

# Enable Lua scripting
lua-time-limit 5000

# Configure script cache
lua-replicate-commands yes

# Set memory limits for Lua scripts
maxmemory-policy allkeys-lru
maxmemory 2gb

# Enable module loading
loadmodule /opt/redis-modules/rejson.so
loadmodule /opt/redis-modules/redistimeseries.so

# Configure logging for debugging
loglevel notice
logfile /var/log/redis/redis-server.log

# Security settings
requirepass your_secure_password_here
protected-mode yes
bind 127.0.0.1 ::1

Install RedisJSON module

Download and compile the RedisJSON module for advanced JSON data processing capabilities.

sudo mkdir -p /opt/redis-modules
cd /tmp
git clone --recursive https://github.com/RedisJSON/RedisJSON.git
cd RedisJSON
make
sudo cp bin/linux-x64-release/rejson.so /opt/redis-modules/
sudo chown redis:redis /opt/redis-modules/rejson.so
sudo chmod 644 /opt/redis-modules/rejson.so

Install RedisTimeSeries module

Install the RedisTimeSeries module for time-series data processing and analytics.

cd /tmp
git clone --recursive https://github.com/RedisTimeSeries/RedisTimeSeries.git
cd RedisTimeSeries
make
sudo cp bin/linux-x64-release/redistimeseries.so /opt/redis-modules/
sudo chown redis:redis /opt/redis-modules/redistimeseries.so
sudo chmod 644 /opt/redis-modules/redistimeseries.so

Create Lua script directory and set permissions

Set up a dedicated directory for Lua scripts with proper ownership and permissions.

sudo mkdir -p /opt/redis-lua-scripts
sudo chown redis:redis /opt/redis-lua-scripts
sudo chmod 755 /opt/redis-lua-scripts

Restart Redis with new configuration

Restart Redis to load the modules and apply the Lua scripting configuration.

sudo systemctl restart redis-server
sudo systemctl status redis-server

Create advanced data processing Lua script

Create a Lua script for batch data processing with error handling and performance optimization.

-- Advanced batch data processor with error handling
local function process_batch(keys, args)
    local batch_size = tonumber(args[1]) or 100
    local operation = args[2] or 'increment'
    local results = {}
    local errors = {}
    
    -- Validate input
    if #keys == 0 then
        return redis.error_reply('No keys provided')
    end
    
    -- Process keys in batches
    for i = 1, #keys, batch_size do
        local batch_end = math.min(i + batch_size - 1, #keys)
        local batch_results = {}
        
        for j = i, batch_end do
            local key = keys[j]
            local success, result = pcall(function()
                if operation == 'increment' then
                    return redis.call('INCR', key)
                elseif operation == 'decrement' then
                    return redis.call('DECR', key)
                elseif operation == 'expire' then
                    local ttl = tonumber(args[3]) or 3600
                    return redis.call('EXPIRE', key, ttl)
                else
                    return redis.call('GET', key)
                end
            end)
            
            if success then
                table.insert(batch_results, {key, result})
            else
                table.insert(errors, {key, result})
            end
        end
        
        table.insert(results, batch_results)
    end
    
    -- Return results and error count
    return {
        processed = #keys,
        errors = #errors,
        results = results,
        error_details = errors
    }
end

return process_batch(KEYS, ARGV)

Create JSON data aggregation script

Create a Lua script that uses RedisJSON for complex data aggregation operations.

-- JSON data aggregation script using RedisJSON
local function aggregate_json_data(keys, args)
    local aggregation_key = args[1]
    local field_path = args[2] or '$'
    local operation = args[3] or 'sum'
    
    local total = 0
    local count = 0
    local values = {}
    
    -- Collect values from JSON objects
    for _, key in ipairs(keys) do
        local exists = redis.call('EXISTS', key)
        if exists == 1 then
            local value = redis.call('JSON.GET', key, field_path)
            if value then
                local num_value = tonumber(value)
                if num_value then
                    table.insert(values, num_value)
                    total = total + num_value
                    count = count + 1
                end
            end
        end
    end
    
    -- Calculate aggregation
    local result
    if operation == 'sum' then
        result = total
    elseif operation == 'avg' and count > 0 then
        result = total / count
    elseif operation == 'min' and #values > 0 then
        result = math.min(table.unpack(values))
    elseif operation == 'max' and #values > 0 then
        result = math.max(table.unpack(values))
    elseif operation == 'count' then
        result = count
    else
        result = 0
    end
    
    -- Store aggregation result
    if aggregation_key then
        redis.call('JSON.SET', aggregation_key, '$', 
            string.format('{
                "operation": "%s",
                "result": %s,
                "count": %d,
                "timestamp": %d
            }', operation, result, count, redis.call('TIME')[1]))
    end
    
    return {
        operation = operation,
        result = result,
        count = count,
        processed_keys = #keys
    }
end

return aggregate_json_data(KEYS, ARGV)

Create time-series data processor

Implement a Lua script for processing time-series data using RedisTimeSeries module.

-- Time-series data processor with RedisTimeSeries
local function process_timeseries(keys, args)
    local retention_period = tonumber(args[1]) or 86400000  -- 24 hours in ms
    local aggregation_window = tonumber(args[2]) or 60000   -- 1 minute in ms
    local current_time = redis.call('TIME')
    local timestamp = current_time[1] * 1000 + math.floor(current_time[2] / 1000)
    
    local processed = 0
    local created = 0
    local errors = {}
    
    for _, key in ipairs(keys) do
        local success, result = pcall(function()
            -- Check if time series exists
            local exists = redis.call('EXISTS', key)
            
            if exists == 0 then
                -- Create new time series with retention and labels
                redis.call('TS.CREATE', key, 
                    'RETENTION', retention_period,
                    'LABELS', 'type', 'metric', 'processed_by', 'lua')
                created = created + 1
            end
            
            -- Add current timestamp with random value (in real use, pass actual values)
            local value = math.random(1, 100)
            redis.call('TS.ADD', key, timestamp, value)
            
            -- Create aggregation rule if it doesn't exist
            local agg_key = key .. ':avg:1m'
            local agg_exists = redis.call('EXISTS', agg_key)
            
            if agg_exists == 0 then
                redis.call('TS.CREATE', agg_key, 
                    'RETENTION', retention_period,
                    'LABELS', 'type', 'aggregation', 'source', key)
                redis.call('TS.CREATERULE', key, agg_key, 
                    'AGGREGATION', 'AVG', aggregation_window)
            end
            
            processed = processed + 1
            return 'OK'
        end)
        
        if not success then
            table.insert(errors, {key, result})
        end
    end
    
    return {
        processed = processed,
        created = created,
        errors = #errors,
        error_details = errors,
        timestamp = timestamp
    }
end

return process_timeseries(KEYS, ARGV)

Load and test Lua scripts

Load the scripts into Redis and test their functionality with sample data.

# Connect to Redis
redis-cli -a your_secure_password_here

# Load batch processor script
SCRIPT LOAD "$(cat /opt/redis-lua-scripts/batch_processor.lua)"

# Load JSON aggregator script  
SCRIPT LOAD "$(cat /opt/redis-lua-scripts/json_aggregator.lua)"

# Load timeseries processor script
SCRIPT LOAD "$(cat /opt/redis-lua-scripts/timeseries_processor.lua)"

# List loaded scripts
SCRIPT LIST

# Test batch processor (create test keys first)
MSET counter1 10 counter2 20 counter3 30
EVALSHA 

Configure performance monitoring

Set up Redis monitoring to track Lua script performance and resource usage.

-- Performance monitoring for Lua scripts
local function get_script_stats(keys, args)
    local stats = {}
    
    -- Get general Redis info
    local info = redis.call('INFO', 'stats')
    local memory_info = redis.call('INFO', 'memory')
    
    -- Get script cache information
    local script_stats = {
        total_calls = redis.call('INFO', 'stats'),
        memory_usage = redis.call('MEMORY', 'USAGE', 'script_cache'),
        cached_scripts = redis.call('SCRIPT', 'LIST')
    }
    
    -- Get module information
    local modules = redis.call('MODULE', 'LIST')
    
    -- Calculate script execution times (estimated)
    local execution_times = {}
    for _, key in ipairs(keys) do
        if redis.call('EXISTS', key) == 1 then
            local start_time = redis.call('TIME')
            -- Simulate script execution time measurement
            execution_times[key] = {
                last_execution = start_time[1] * 1000000 + start_time[2],
                avg_time = math.random(1, 1000)  -- In microseconds
            }
        end
    end
    
    return {
        timestamp = redis.call('TIME')[1],
        script_stats = script_stats,
        modules = modules,
        execution_times = execution_times,
        memory_used = redis.call('INFO', 'memory')
    }
end

return get_script_stats(KEYS, ARGV)

Create script management utility

Create a Python script to manage and monitor Lua scripts in production environments.

#!/usr/bin/env python3
import redis
import hashlib
import json
import time
import os
from typing import Dict, List, Any

class RedisLuaManager:
    def __init__(self, host='localhost', port=6379, password=None, db=0):
        self.redis_client = redis.Redis(
            host=host, port=port, password=password, db=db, 
            decode_responses=True, socket_timeout=5
        )
        self.script_cache = {}
    
    def load_script_from_file(self, script_path: str) -> str:
        """Load and register a Lua script from file"""
        try:
            with open(script_path, 'r') as f:
                script_content = f.read()
            
            sha = self.redis_client.script_load(script_content)
            script_name = os.path.basename(script_path)
            self.script_cache[script_name] = {
                'sha': sha,
                'path': script_path,
                'loaded_at': time.time()
            }
            
            print(f"Loaded script {script_name}: {sha}")
            return sha
        except Exception as e:
            print(f"Error loading script {script_path}: {e}")
            return None
    
    def execute_script(self, script_name: str, keys: List[str] = None, args: List[str] = None) -> Any:
        """Execute a loaded script by name"""
        if script_name not in self.script_cache:
            raise ValueError(f"Script {script_name} not loaded")
        
        sha = self.script_cache[script_name]['sha']
        keys = keys or []
        args = args or []
        
        try:
            start_time = time.time()
            result = self.redis_client.evalsha(sha, len(keys), *(keys + args))
            execution_time = (time.time() - start_time) * 1000
            
            print(f"Script {script_name} executed in {execution_time:.2f}ms")
            return result
        except redis.exceptions.NoScriptError:
            print(f"Script {script_name} not in cache, reloading...")
            self.load_script_from_file(self.script_cache[script_name]['path'])
            return self.execute_script(script_name, keys, args)
    
    def get_script_stats(self) -> Dict[str, Any]:
        """Get performance statistics for all scripts"""
        info = self.redis_client.info()
        return {
            'total_scripts': len(self.script_cache),
            'redis_memory_used': info.get('used_memory_human'),
            'redis_connected_clients': info.get('connected_clients'),
            'script_cache': self.script_cache
        }
    
    def benchmark_script(self, script_name: str, iterations: int = 100, 
                        keys: List[str] = None, args: List[str] = None) -> Dict[str, float]:
        """Benchmark script execution performance"""
        execution_times = []
        
        for _ in range(iterations):
            start_time = time.time()
            try:
                self.execute_script(script_name, keys, args)
                execution_times.append((time.time() - start_time) * 1000)
            except Exception as e:
                print(f"Benchmark error: {e}")
        
        if execution_times:
            return {
                'avg_time': sum(execution_times) / len(execution_times),
                'min_time': min(execution_times),
                'max_time': max(execution_times),
                'total_iterations': len(execution_times)
            }
        return {}

if __name__ == '__main__':
    manager = RedisLuaManager(password='your_secure_password_here')
    
    # Load all scripts
    script_dir = '/opt/redis-lua-scripts'
    for script_file in ['batch_processor.lua', 'json_aggregator.lua', 
                       'timeseries_processor.lua', 'monitor.lua']:
        script_path = os.path.join(script_dir, script_file)
        if os.path.exists(script_path):
            manager.load_script_from_file(script_path)
    
    # Example usage
    print(json.dumps(manager.get_script_stats(), indent=2))
    
    # Benchmark example
    benchmark = manager.benchmark_script('batch_processor.lua', 
                                        iterations=50,
                                        keys=['test1', 'test2', 'test3'],
                                        args=['10', 'increment'])
    print(f"Benchmark results: {benchmark}")

Set up script monitoring with systemd

Create a systemd service to monitor script performance and automatically reload scripts if needed.

[Unit]
Description=Redis Lua Script Monitor
After=redis-server.service
Requires=redis-server.service

[Service]
Type=simple
User=redis
Group=redis
WorkingDirectory=/opt/redis-lua-scripts
ExecStart=/usr/bin/python3 /opt/redis-lua-scripts/script_manager.py
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target

sudo chmod 644 /opt/redis-lua-scripts/script_manager.py
sudo chmod +x /opt/redis-lua-scripts/script_manager.py
sudo systemctl daemon-reload
sudo systemctl enable redis-script-monitor
sudo systemctl start redis-script-monitor

Configure Lua script caching

Optimize script caching and memory usage for production workloads.

# Script cache optimization
lua-replicate-commands yes
lua-always-replicate-commands no

# Memory optimization for scripts
maxmemory-policy allkeys-lru
lua-time-limit