Set up comprehensive Istio security policies with external authorization services, JWT validation, and RBAC integration for enterprise-grade service mesh security and compliance.
Prerequisites
- Kubernetes cluster with Istio installed
- kubectl and istioctl CLI tools
- External authorization service (OPA)
- Valid JWT provider (Keycloak/Auth0)
- Prometheus and Grafana for monitoring
What this solves
Istio's built-in security features need external authorization for complex enterprise requirements like fine-grained access control, audit compliance, and integration with existing identity providers. This tutorial shows how to configure Istio AuthorizationPolicy with external authorization services, implement JWT token validation with RBAC, and monitor authorization decisions in production environments.
You'll integrate Open Policy Agent (OPA) as an external authorization service, configure JWT validation with Keycloak, and set up comprehensive monitoring for authorization policies across your service mesh.
Step-by-step configuration
Update system packages
Start by updating your system and installing required dependencies for the authorization service setup.
sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget jq git
Install kubectl and istioctl
Install the Kubernetes and Istio command-line tools for managing your service mesh configuration.
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
curl -L https://istio.io/downloadIstio | sh -
sudo mv istio-*/bin/istioctl /usr/local/bin/
rm -rf istio-*
Deploy Open Policy Agent as external authorization service
Deploy OPA as a sidecar service that will handle external authorization decisions for your service mesh.
apiVersion: apps/v1
kind: Deployment
metadata:
name: opa-authz
namespace: istio-system
labels:
app: opa-authz
spec:
replicas: 2
selector:
matchLabels:
app: opa-authz
template:
metadata:
labels:
app: opa-authz
version: v1
spec:
containers:
- name: opa
image: openpolicyagent/opa:0.58.0-envoy
ports:
- containerPort: 8181
- containerPort: 9191
args:
- "run"
- "--server"
- "--config-file=/config/config.yaml"
- "--addr=localhost:8181"
- "--diagnostic-addr=0.0.0.0:8282"
- "/policies"
volumeMounts:
- readOnly: true
mountPath: /config
name: opa-config
- readOnly: true
mountPath: /policies
name: opa-policy
livenessProbe:
httpGet:
path: /health?plugins
port: 8282
initialDelaySeconds: 5
periodSeconds: 5
readinessProbe:
httpGet:
path: /health?plugins
port: 8282
initialDelaySeconds: 5
periodSeconds: 5
volumes:
- name: opa-config
configMap:
name: opa-config
- name: opa-policy
configMap:
name: opa-policy
---
apiVersion: v1
kind: Service
metadata:
name: opa-authz
namespace: istio-system
labels:
app: opa-authz
spec:
selector:
app: opa-authz
ports:
- name: grpc
port: 9191
targetPort: 9191
- name: http
port: 8181
targetPort: 8181
Create OPA configuration
Configure OPA to work as an external authorization service with Envoy proxy integration.
apiVersion: v1
kind: ConfigMap
metadata:
name: opa-config
namespace: istio-system
data:
config.yaml: |
plugins:
envoy_ext_authz_grpc:
addr: :9191
query: data.envoy.authz.allow
enable_reflection: true
decision_logs:
console: true
status:
console: true
bundles:
authz:
resource: "/policies/policy.rego"
kubectl apply -f opa-config.yaml
Create authorization policy rules
Define OPA policies that will evaluate authorization requests based on JWT claims, HTTP headers, and service metadata.
apiVersion: v1
kind: ConfigMap
metadata:
name: opa-policy
namespace: istio-system
data:
policy.rego: |
package envoy.authz
import future.keywords.if
import future.keywords.in
default allow := false
# Allow health checks
allow if {
input.attributes.request.http.path == "/health"
}
# Allow requests with valid JWT and proper roles
allow if {
jwt_valid
has_required_role
rate_limit_ok
}
# JWT validation
jwt_valid if {
jwt := input.attributes.request.http.headers.authorization
startswith(jwt, "Bearer ")
token := substring(jwt, 7, -1)
payload := io.jwt.decode_verify(token, {
"cert": jwks_cert,
"aud": "istio-demo",
"iss": "https://keycloak.example.com/realms/demo"
})
payload[1].exp > time.now_ns() / 1000000000
}
# Role-based access control
has_required_role if {
jwt := input.attributes.request.http.headers.authorization
token := substring(jwt, 7, -1)
payload := io.jwt.decode(token)
claims := payload[1]
required_roles := service_roles[input.attributes.destination.service.name]
count(claims.realm_access.roles & required_roles) > 0
}
# Rate limiting check
rate_limit_ok if {
user_id := get_user_id
request_count := count_recent_requests(user_id)
request_count < 100 # 100 requests per minute
}
# Service role mappings
service_roles := {
"productcatalog": {"user", "admin"},
"cartservice": {"user", "admin"},
"paymentservice": {"admin"},
"adminservice": {"admin"}
}
# Extract user ID from JWT
get_user_id := claims.sub if {
jwt := input.attributes.request.http.headers.authorization
token := substring(jwt, 7, -1)
payload := io.jwt.decode(token)
claims := payload[1]
}
# JWKS certificate (replace with your actual certificate)
jwks_cert := `-----BEGIN CERTIFICATE-----
MIIC...your-cert-here...==
-----END CERTIFICATE-----`
# Helper function for rate limiting (simplified)
count_recent_requests(user_id) := 0 # Implement actual rate limiting logic
kubectl apply -f opa-policy.yaml
Deploy the OPA authorization service
Apply the OPA deployment configuration and verify it's running correctly.
kubectl apply -f opa-authz.yaml
kubectl get pods -n istio-system -l app=opa-authz
kubectl logs -n istio-system -l app=opa-authz --tail=50
Configure Istio ExtensionProvider
Register the OPA service as an external authorization provider in Istio's configuration.
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
metadata:
name: control-plane
namespace: istio-system
spec:
meshConfig:
extensionProviders:
- name: opa-authz
envoyExtAuthzGrpc:
service: opa-authz.istio-system.svc.cluster.local
port: 9191
timeout: 3s
includeRequestHeadersInCheck:
- authorization
- x-user-id
- x-forwarded-for
includeAdditionalHeadersInCheck:
x-ext-authz-check-result: "true"
- name: otel
envoyOtelAls:
service: jaeger-collector.istio-system.svc.cluster.local
port: 4317
values:
telemetry:
v2:
prometheus:
configOverride:
metric_relabeling_configs:
- source_labels: [__name__]
regex: 'istio_request_total'
target_label: __tmp_istio_request_total
- source_labels: [__tmp_istio_request_total]
target_label: __name__
replacement: 'istio_requests_total'
kubectl apply -f extensionprovider-config.yaml
Create AuthorizationPolicy with external authorization
Configure Istio to use the external OPA service for authorization decisions on specific workloads.
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: external-authz
namespace: production
spec:
selector:
matchLabels:
app: productcatalog
action: CUSTOM
provider:
name: opa-authz
rules:
- to:
- operation:
methods: ["GET", "POST", "PUT", "DELETE"]
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: payment-service-authz
namespace: production
spec:
selector:
matchLabels:
app: paymentservice
action: CUSTOM
provider:
name: opa-authz
rules:
- to:
- operation:
methods: ["POST", "PUT"]
paths: ["/api/v1/payments/*"]
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: admin-service-authz
namespace: production
spec:
selector:
matchLabels:
app: adminservice
action: CUSTOM
provider:
name: opa-authz
rules:
- to:
- operation:
methods: ["GET", "POST", "PUT", "DELETE"]
paths: ["/admin/*"]
kubectl apply -f authz-policy.yaml
Configure JWT validation policy
Set up Istio RequestAuthentication to validate JWT tokens before they reach the external authorization service.
apiVersion: security.istio.io/v1beta1
kind: RequestAuthentication
metadata:
name: jwt-auth
namespace: production
spec:
selector:
matchLabels:
version: v1
jwtRules:
- issuer: "https://keycloak.example.com/realms/demo"
jwksUri: "https://keycloak.example.com/realms/demo/protocol/openid-connect/certs"
audiences:
- "istio-demo"
forwardOriginalToken: true
outputPayloadToHeader: "x-jwt-payload"
- issuer: "https://auth0.example.com/"
jwksUri: "https://auth0.example.com/.well-known/jwks.json"
audiences:
- "api://default"
fromHeaders:
- name: "x-access-token"
- name: "authorization"
prefix: "Bearer "
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: require-jwt
namespace: production
spec:
selector:
matchLabels:
app: productcatalog
action: ALLOW
rules:
- from:
- source:
requestPrincipals: ["https://keycloak.example.com/realms/demo/*"]
- from:
- source:
requestPrincipals: ["https://auth0.example.com/*"]
- to:
- operation:
methods: ["GET"]
paths: ["/health", "/ready"]
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: deny-without-jwt
namespace: production
spec:
selector:
matchLabels:
app: paymentservice
action: DENY
rules:
- from:
- source:
notRequestPrincipals: ["*"]
to:
- operation:
notMethods: ["OPTIONS"]
kubectl apply -f jwt-policy.yaml
Enable telemetry for authorization monitoring
Configure Istio to collect detailed metrics and traces for authorization decisions and policy enforcement.
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
name: authz-metrics
namespace: istio-system
spec:
metrics:
- providers:
- name: prometheus
- overrides:
- match:
metric: ALL_METRICS
tagOverrides:
source_service_name:
operation: UPSERT
value: "%{SOURCE_WORKLOAD}-%{SOURCE_NAMESPACE}"
destination_service_name:
operation: UPSERT
value: "%{DESTINATION_SERVICE_NAME}"
authz_result:
operation: UPSERT
value: "%{RESPONSE_CODE}"
---
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
name: authz-access-logs
namespace: production
spec:
accessLogging:
- providers:
- name: otel
- match:
mode: CLIENT
filter:
expression: 'response.code >= 400 || response.code == 200'
format:
labels:
source_app: "%{SOURCE_APP}"
dest_app: "%{DESTINATION_APP}"
method: "%{REQUEST_METHOD}"
path: "%{REQUEST_URL_PATH}"
user_id: "%{REQUEST_HEADERS['x-user-id']}"
authz_duration: "%{RESPONSE_HEADERS['x-ext-authz-duration']}"
jwt_claims: "%{REQUEST_HEADERS['x-jwt-payload']}"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: istio-grafana-dashboards
namespace: istio-system
labels:
grafana_dashboard: "1"
data:
authz-dashboard.json: |
{
"dashboard": {
"title": "Istio Authorization Policies",
"panels": [
{
"title": "Authorization Success Rate",
"type": "stat",
"targets": [{
"expr": "sum(rate(istio_requests_total{response_code!~'4..|5..'}[5m])) / sum(rate(istio_requests_total[5m]))",
"legendFormat": "Success Rate"
}]
},
{
"title": "Authorization Denials by Service",
"type": "bargauge",
"targets": [{
"expr": "sum(rate(istio_requests_total{response_code=~'403|401'}[5m])) by (destination_service_name)",
"legendFormat": "{{destination_service_name}}"
}]
},
{
"title": "External Authorization Latency",
"type": "graph",
"targets": [{
"expr": "histogram_quantile(0.95, sum(rate(envoy_ext_authz_duration_bucket[5m])) by (le))",
"legendFormat": "95th percentile"
}]
}
]
}
}
kubectl apply -f authz-telemetry.yaml
Deploy sample applications for testing
Deploy test applications to validate the authorization policies are working correctly.
apiVersion: apps/v1
kind: Deployment
metadata:
name: productcatalog
namespace: production
spec:
replicas: 2
selector:
matchLabels:
app: productcatalog
version: v1
template:
metadata:
labels:
app: productcatalog
version: v1
annotations:
sidecar.istio.io/inject: "true"
spec:
containers:
- name: productcatalog
image: gcr.io/google-samples/microservices-demo/productcatalogservice:v0.8.0
ports:
- containerPort: 3550
env:
- name: PORT
value: "3550"
readinessProbe:
httpGet:
path: /health
port: 3550
initialDelaySeconds: 10
livenessProbe:
httpGet:
path: /health
port: 3550
initialDelaySeconds: 10
---
apiVersion: v1
kind: Service
metadata:
name: productcatalog
namespace: production
spec:
selector:
app: productcatalog
ports:
- port: 3550
targetPort: 3550
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: paymentservice
namespace: production
spec:
replicas: 1
selector:
matchLabels:
app: paymentservice
version: v1
template:
metadata:
labels:
app: paymentservice
version: v1
annotations:
sidecar.istio.io/inject: "true"
spec:
containers:
- name: paymentservice
image: gcr.io/google-samples/microservices-demo/paymentservice:v0.8.0
ports:
- containerPort: 50051
env:
- name: PORT
value: "50051"
readinessProbe:
grpc:
port: 50051
initialDelaySeconds: 10
livenessProbe:
grpc:
port: 50051
initialDelaySeconds: 10
---
apiVersion: v1
kind: Service
metadata:
name: paymentservice
namespace: production
spec:
selector:
app: paymentservice
ports:
- port: 50051
targetPort: 50051
kubectl create namespace production
kubectl label namespace production istio-injection=enabled
kubectl apply -f test-apps.yaml
Set up monitoring and alerting
Configure Prometheus rules and Grafana dashboards to monitor authorization policy performance and security events.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: istio-authz-alerts
namespace: istio-system
labels:
prometheus: istio-proxy
role: alert-rules
spec:
groups:
- name: istio-authz
rules:
- alert: HighAuthorizationFailureRate
expr: |
(
sum(rate(istio_requests_total{response_code=~"403|401"}[5m]))
/
sum(rate(istio_requests_total[5m]))
) > 0.1
for: 2m
labels:
severity: warning
annotations:
summary: "High authorization failure rate detected"
description: "Authorization failure rate is {{ $value | humanizePercentage }} which is above the 10% threshold"
- alert: ExternalAuthzServiceDown
expr: up{job="opa-authz"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: "External authorization service is down"
description: "OPA authorization service has been down for more than 1 minute"
- alert: HighAuthzLatency
expr: |
histogram_quantile(0.95,
sum(rate(envoy_ext_authz_duration_bucket[5m])) by (le)
) > 1.0
for: 5m
labels:
severity: warning
annotations:
summary: "High external authorization latency"
description: "95th percentile authorization latency is {{ $value }}s which is above 1s threshold"
- alert: JWTValidationErrors
expr: |
sum(rate(istio_requests_total{response_code="401",response_flags=~".UAEX."}[5m])) > 0.5
for: 2m
labels:
severity: warning
annotations:
summary: "High JWT validation error rate"
description: "JWT validation errors are occurring at {{ $value }} requests/sec"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: authz-grafana-dashboard
namespace: istio-system
labels:
grafana_dashboard: "1"
data:
istio-authz.json: |
{
"dashboard": {
"id": null,
"title": "Istio External Authorization",
"tags": ["istio", "security"],
"timezone": "browser",
"panels": [
{
"id": 1,
"title": "Authorization Decision Rate",
"type": "graph",
"targets": [
{
"expr": "sum(rate(istio_requests_total[5m])) by (response_code)",
"legendFormat": "{{response_code}}"
}
]
},
{
"id": 2,
"title": "Top Denied Services",
"type": "table",
"targets": [
{
"expr": "topk(10, sum(rate(istio_requests_total{response_code=~'403|401'}[5m])) by (destination_service_name, source_app))",
"format": "table"
}
]
},
{
"id": 3,
"title": "External Authorization Latency Distribution",
"type": "heatmap",
"targets": [
{
"expr": "sum(rate(envoy_ext_authz_duration_bucket[5m])) by (le)",
"format": "heatmap"
}
]
}
],
"time": {
"from": "now-1h",
"to": "now"
},
"refresh": "30s"
}
}
kubectl apply -f authz-monitoring.yaml
Verify your setup
Test the authorization policies with different scenarios to ensure they're working correctly.
# Check OPA service status
kubectl get pods -n istio-system -l app=opa-authz
kubectl logs -n istio-system -l app=opa-authz --tail=20
# Test unauthorized access (should return 403)
curl -v http://productcatalog.production.svc.cluster.local:3550/products
Test with valid JWT token (replace with actual token)
export JWT_TOKEN="eyJhbGciOiJSUzI1NiIs..."
curl -H "Authorization: Bearer $JWT_TOKEN" \
http://productcatalog.production.svc.cluster.local:3550/products
# Check authorization policies are applied
kubectl get authorizationpolicy -n production
kubectl describe authorizationpolicy external-authz -n production
# Verify telemetry is collecting authorization metrics
kubectl exec -n istio-system deploy/prometheus -- \
promtool query instant 'sum(rate(istio_requests_total[5m])) by (response_code)'
Monitor and troubleshoot authorization policies
Set up comprehensive logging
Configure detailed access logging to track authorization decisions and debug policy issues.
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
name: authz-debug-logs
namespace: production
spec:
accessLogging:
- providers:
- name: envoy
format:
text: |
[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
%RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT%
%DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%"
"%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"
"%REQ(AUTHORIZATION)%" ext_authz_status=%DYNAMIC_METADATA(envoy.ext_authz:status)%
ext_authz_duration=%DYNAMIC_METADATA(envoy.ext_authz:duration)%
kubectl apply -f authz-debug-logging.yaml
Create troubleshooting scripts
Set up automated scripts to diagnose common authorization issues and policy misconfigurations.
#!/bin/bash
Debug script for Istio authorization policies
set -e
NAMESPACE="production"
OPA_NAMESPACE="istio-system"
echo "=== Istio Authorization Debug Report ==="
echo "Timestamp: $(date)"
echo
echo "1. Checking OPA External Authorization Service:"
kubectl get pods -n $OPA_NAMESPACE -l app=opa-authz -o wide
echo
echo "2. OPA Service Health:"
kubectl get svc -n $OPA_NAMESPACE opa-authz
echo
echo "3. Recent OPA Logs (errors and decisions):"
kubectl logs -n $OPA_NAMESPACE -l app=opa-authz --tail=50 | grep -E "ERROR|decision|allow|deny" || echo "No relevant logs found"
echo
echo "4. Authorization Policies Status:"
kubectl get authorizationpolicy -A
echo
echo "5. Request Authentication Policies:"
kubectl get requestauthentication -A
echo
echo "6. Recent Authorization Denials (403/401):"
kubectl logs -n istio-system -l app=istiod --tail=200 | grep -E "403|401|denied" | tail -10 || echo "No recent denials in istiod logs"
echo
echo "7. Envoy Proxy Configuration for External Authorization:"
for pod in $(kubectl get pods -n $NAMESPACE -l security.istio.io/tlsMode=istio -o jsonpath='{.items[*].metadata.name}'); do
echo "Checking pod: $pod"
kubectl exec -n $NAMESPACE $pod -c istio-proxy -- curl -s localhost:15000/config_dump | \
jq '.configs[] | select(."@type" | contains("type.googleapis.com/envoy.admin.v3.ClustersConfigDump")) | .dynamic_active_clusters[] | select(.cluster.name | contains("ext-authz"))' 2>/dev/null || echo "No ext-authz cluster found"
echo
done
echo "8. Testing Connectivity to OPA Service:"
kubectl run test-authz --image=curlimages/curl --rm -i --restart=Never -- \
curl -v -m 5 opa-authz.istio-system.svc.cluster.local:8181/health 2>&1 || echo "Connection test failed"
echo "=== Debug Report Complete ==="
chmod +x debug-authz.sh
./debug-authz.sh
Set up policy testing framework
Create automated tests to validate authorization policies work as expected before deploying to production.
apiVersion: v1
kind: ConfigMap
metadata:
name: authz-tests
namespace: production
data:
test-suite.sh: |
#!/bin/bash
set -e
# Test configuration
PRODUCT_SERVICE="http://productcatalog.production.svc.cluster.local:3550"
PAYMENT_SERVICE="http://paymentservice.production.svc.cluster.local:50051"
VALID_JWT="eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..."
EXPIRED_JWT="eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..."
INVALID_JWT="invalid.jwt.token"
# Test counters
PASSED=0
FAILED=0
test_case() {
local description="$1"
local expected_code="$2"
local actual_code="$3"
echo -n "Testing: $description... "
if [ "$expected_code" = "$actual_code" ]; then
echo "PASS"
((PASSED++))
else
echo "FAIL (expected $expected_code, got $actual_code)"
((FAILED++))
fi
}
echo "=== Authorization Policy Test Suite ==="
echo "Starting tests at $(date)"
echo
# Test 1: Unauthorized access should be denied
CODE=$(curl -s -o /dev/null -w "%{http_code}" $PRODUCT_SERVICE/products || echo "000")
test_case "Unauthorized access to product catalog" "403" "$CODE"
# Test 2: Valid JWT should allow access
CODE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer $VALID_JWT" \
$PRODUCT_SERVICE/products || echo "000")
test_case "Authorized access with valid JWT" "200" "$CODE"
# Test 3: Expired JWT should be denied
CODE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer $EXPIRED_JWT" \
$PRODUCT_SERVICE/products || echo "000")
test_case "Access with expired JWT" "401" "$CODE"
# Test 4: Invalid JWT should be denied
CODE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer $INVALID_JWT" \
$PRODUCT_SERVICE/products || echo "000")
test_case "Access with invalid JWT" "401" "$CODE"
# Test 5: Health endpoints should always be accessible
CODE=$(curl -s -o /dev/null -w "%{http_code}" \
$PRODUCT_SERVICE/health || echo "000")
test_case "Health endpoint access without auth" "200" "$CODE"
# Test 6: Payment service should require admin role
CODE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer $VALID_JWT" \
-X POST $PAYMENT_SERVICE/api/v1/payments || echo "000")
test_case "Payment service with user token (should fail)" "403" "$CODE"
echo
echo "=== Test Results ==="
echo "PASSED: $PASSED"
echo "FAILED: $FAILED"
echo "TOTAL: $((PASSED + FAILED))"
if [ $FAILED -gt 0 ]; then
echo "Some tests failed. Check authorization policies and OPA rules."
exit 1
else
echo "All tests passed!"
exit 0
fi
---
apiVersion: batch/v1
kind: Job
metadata:
name: authz-policy-tests
namespace: production
spec:
template:
spec:
containers:
- name: test-runner
image: curlimages/curl:latest
command: ["/bin/sh"]
args: ["/scripts/test-suite.sh"]
volumeMounts:
- name: test-scripts
mountPath: /scripts
volumes:
- name: test-scripts
configMap:
name: authz-tests
defaultMode: 0755
restartPolicy: Never
backoffLimit: 3
kubectl apply -f test-authz-policies.yaml
kubectl logs job/authz-policy-tests -n production -f
Common issues
| Symptom | Cause | Fix |
|---|---|---|
| All requests return 500 errors | OPA service is down or misconfigured | Check kubectl logs -n istio-system -l app=opa-authz and verify service connectivity |
| Valid JWTs are being rejected | JWKS URI unreachable or certificate mismatch | Verify jwksUri in RequestAuthentication and check OPA policy certificate |
| Authorization policies not applying | Label selectors don't match pods | Confirm pod labels match AuthorizationPolicy spec.selector |
| External authorization taking too long | OPA policy is too complex or has infinite loops | Simplify policy rules and add timeout configuration in Istio |
| Policies work intermittently | OPA pods are restarting due to resource limits | Increase CPU/memory requests and add proper resource limits |
| Metrics not showing in Grafana | Prometheus not scraping Istio metrics correctly | Verify ServiceMonitor configuration and check Prometheus targets |
Next steps
- Implement Istio security scanning and vulnerability management for Kubernetes service mesh
- Configure Istio distributed tracing with Jaeger and Zipkin for comprehensive microservices observability
- Implement Kubernetes secrets management with HashiCorp Vault integration
- Configure Istio custom authorization policies with Lua scripting
- Implement Istio security policies with workload identity federation
Running this in production?
Automated install script
Run this to automate the entire setup
#!/usr/bin/env bash
set -euo pipefail
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
NC='\033[0m' # No Color
# Global variables
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_FILE="/var/log/istio-authz-install.log"
TEMP_DIR=$(mktemp -d)
# Usage function
usage() {
echo "Usage: $0 [OPTIONS]"
echo "Options:"
echo " -k, --keycloak-url URL Keycloak server URL (default: http://keycloak:8080)"
echo " -d, --domain DOMAIN Domain for JWT issuer (default: example.com)"
echo " -h, --help Show this help message"
exit 1
}
# Logging function
log() {
local level=$1
shift
local message="$*"
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "${level}[${timestamp}] ${message}${NC}" | tee -a "$LOG_FILE"
}
# Error handling
cleanup() {
if [ -d "$TEMP_DIR" ]; then
rm -rf "$TEMP_DIR"
fi
}
error_exit() {
log "$RED" "ERROR: $1"
cleanup
exit 1
}
trap cleanup EXIT
trap 'error_exit "Script failed on line $LINENO"' ERR
# Parse command line arguments
KEYCLOAK_URL="http://keycloak:8080"
JWT_DOMAIN="example.com"
while [[ $# -gt 0 ]]; do
case $1 in
-k|--keycloak-url)
KEYCLOAK_URL="$2"
shift 2
;;
-d|--domain)
JWT_DOMAIN="$2"
shift 2
;;
-h|--help)
usage
;;
*)
echo "Unknown option: $1"
usage
;;
esac
done
# Check if running as root or with sudo
check_privileges() {
if [[ $EUID -ne 0 ]] && ! sudo -n true 2>/dev/null; then
error_exit "This script requires root privileges or sudo access"
fi
}
# Auto-detect distribution
detect_distro() {
if [ -f /etc/os-release ]; then
. /etc/os-release
case "$ID" in
ubuntu|debian)
PKG_MGR="apt"
PKG_UPDATE="apt update"
PKG_INSTALL="apt install -y"
;;
almalinux|rocky|centos|rhel|ol|fedora)
PKG_MGR="dnf"
PKG_UPDATE="dnf update -y --refresh"
PKG_INSTALL="dnf install -y"
;;
amzn)
PKG_MGR="yum"
PKG_UPDATE="yum update -y"
PKG_INSTALL="yum install -y"
;;
*)
error_exit "Unsupported distribution: $ID"
;;
esac
else
error_exit "Cannot detect distribution - /etc/os-release not found"
fi
log "$GREEN" "Detected distribution: $PRETTY_NAME"
}
# Check prerequisites
check_prerequisites() {
log "$YELLOW" "[1/8] Checking prerequisites..."
# Check for kubectl
if ! command -v kubectl &> /dev/null; then
error_exit "kubectl is required but not installed"
fi
# Check for kubernetes cluster connectivity
if ! kubectl cluster-info &> /dev/null; then
error_exit "Cannot connect to Kubernetes cluster"
fi
# Check for istio-system namespace
if ! kubectl get namespace istio-system &> /dev/null; then
error_exit "istio-system namespace not found. Please install Istio first"
fi
}
# Update system packages
update_system() {
log "$YELLOW" "[2/8] Updating system packages..."
sudo $PKG_UPDATE
sudo $PKG_INSTALL curl wget jq git
}
# Install kubectl and istioctl
install_tools() {
log "$YELLOW" "[3/8] Installing kubectl and istioctl..."
# Install kubectl
if ! command -v kubectl &> /dev/null; then
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
chmod 755 kubectl
sudo mv kubectl /usr/local/bin/
fi
# Install istioctl
if ! command -v istioctl &> /dev/null; then
curl -L https://istio.io/downloadIstio | sh -
sudo mv istio-*/bin/istioctl /usr/local/bin/
chmod 755 /usr/local/bin/istioctl
rm -rf istio-*
fi
}
# Create OPA configuration
create_opa_config() {
log "$YELLOW" "[4/8] Creating OPA configuration..."
cat > "$TEMP_DIR/opa-config.yaml" << 'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
name: opa-config
namespace: istio-system
data:
config.yaml: |
plugins:
envoy_ext_authz_grpc:
addr: :9191
query: data.envoy.authz.allow
enable_reflection: true
decision_logs:
console: true
status:
console: true
bundles:
authz:
resource: "/policies/policy.rego"
EOF
kubectl apply -f "$TEMP_DIR/opa-config.yaml"
}
# Create OPA policy
create_opa_policy() {
log "$YELLOW" "[5/8] Creating OPA authorization policies..."
cat > "$TEMP_DIR/opa-policy.yaml" << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: opa-policy
namespace: istio-system
data:
policy.rego: |
package envoy.authz
import future.keywords.if
import future.keywords.in
default allow := false
# Allow health checks
allow if {
input.attributes.request.http.path == "/health"
}
# Allow requests with valid JWT tokens
allow if {
valid_jwt
authorized_user
}
# JWT validation
valid_jwt if {
token := extract_jwt_token
jwt.verify_rs256(token, jwks_key)
}
# Extract JWT token from Authorization header
extract_jwt_token := token if {
auth_header := input.attributes.request.http.headers.authorization
startswith(auth_header, "Bearer ")
token := substring(auth_header, 7, -1)
}
# Check user authorization based on roles
authorized_user if {
token := extract_jwt_token
payload := jwt.decode_verify(token, {"iss": "${KEYCLOAK_URL}/auth/realms/${JWT_DOMAIN}"})
"admin" in payload[2].realm_access.roles
}
authorized_user if {
token := extract_jwt_token
payload := jwt.decode_verify(token, {"iss": "${KEYCLOAK_URL}/auth/realms/${JWT_DOMAIN}"})
"user" in payload[2].realm_access.roles
input.attributes.request.http.method == "GET"
}
# Mock JWKS for demonstration
jwks_key := {
"kty": "RSA",
"use": "sig",
"alg": "RS256"
}
EOF
kubectl apply -f "$TEMP_DIR/opa-policy.yaml"
}
# Deploy OPA authorization service
deploy_opa_service() {
log "$YELLOW" "[6/8] Deploying OPA authorization service..."
cat > "$TEMP_DIR/opa-deployment.yaml" << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: opa-authz
namespace: istio-system
labels:
app: opa-authz
spec:
replicas: 2
selector:
matchLabels:
app: opa-authz
template:
metadata:
labels:
app: opa-authz
version: v1
spec:
containers:
- name: opa
image: openpolicyagent/opa:0.58.0-envoy
ports:
- containerPort: 8181
- containerPort: 9191
args:
- "run"
- "--server"
- "--config-file=/config/config.yaml"
- "--addr=localhost:8181"
- "--diagnostic-addr=0.0.0.0:8282"
- "/policies"
volumeMounts:
- readOnly: true
mountPath: /config
name: opa-config
- readOnly: true
mountPath: /policies
name: opa-policy
livenessProbe:
httpGet:
path: /health?plugins
port: 8282
initialDelaySeconds: 5
periodSeconds: 5
readinessProbe:
httpGet:
path: /health?plugins
port: 8282
initialDelaySeconds: 5
periodSeconds: 5
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "200m"
volumes:
- name: opa-config
configMap:
name: opa-config
- name: opa-policy
configMap:
name: opa-policy
---
apiVersion: v1
kind: Service
metadata:
name: opa-authz
namespace: istio-system
labels:
app: opa-authz
spec:
selector:
app: opa-authz
ports:
- name: grpc
port: 9191
targetPort: 9191
- name: http
port: 8181
targetPort: 8181
EOF
kubectl apply -f "$TEMP_DIR/opa-deployment.yaml"
}
# Configure Istio authorization policy
configure_istio_authz() {
log "$YELLOW" "[7/8] Configuring Istio authorization policy..."
cat > "$TEMP_DIR/authz-policy.yaml" << 'EOF'
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: ext-authz
namespace: istio-system
spec:
selector:
matchLabels:
app: opa-authz
action: CUSTOM
provider:
name: "opa-authz"
rules:
- to:
- operation:
methods: ["GET", "POST", "PUT", "DELETE"]
---
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
metadata:
name: external-authz
spec:
meshConfig:
extensionProviders:
- name: "opa-authz"
envoyExtAuthzGrpc:
service: "opa-authz.istio-system.svc.cluster.local"
port: "9191"
EOF
kubectl apply -f "$TEMP_DIR/authz-policy.yaml"
}
# Verify installation
verify_installation() {
log "$YELLOW" "[8/8] Verifying installation..."
# Check OPA deployment
if kubectl rollout status deployment/opa-authz -n istio-system --timeout=300s; then
log "$GREEN" "OPA authorization service deployed successfully"
else
error_exit "OPA deployment failed"
fi
# Check OPA service
if kubectl get svc opa-authz -n istio-system &> /dev/null; then
log "$GREEN" "OPA service created successfully"
else
error_exit "OPA service not found"
fi
# Check authorization policy
if kubectl get authorizationpolicy ext-authz -n istio-system &> /dev/null; then
log "$GREEN" "Authorization policy created successfully"
else
error_exit "Authorization policy not found"
fi
log "$GREEN" "Installation completed successfully!"
log "$GREEN" "OPA external authorization service is now configured with Istio"
log "$YELLOW" "Next steps:"
echo "1. Configure your Keycloak server at: $KEYCLOAK_URL"
echo "2. Create realm: $JWT_DOMAIN"
echo "3. Configure JWT issuer and JWKS endpoint"
echo "4. Apply authorization policies to your services"
}
# Main execution
main() {
log "$GREEN" "Starting Istio external authorization configuration..."
check_privileges
detect_distro
check_prerequisites
update_system
install_tools
create_opa_config
create_opa_policy
deploy_opa_service
configure_istio_authz
verify_installation
log "$GREEN" "Istio external authorization setup completed successfully!"
}
main "$@"
Review the script before running. Execute with: bash install.sh