Kubernetes Policy Engines

Install OPA Gatekeeper

# Install using kubectl
kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper/release-3.14/deploy/gatekeeper.yaml

# Or install using Helm
helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts
helm install gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system \
  --create-namespace

# Verify installation
kubectl get pods -n gatekeeper-system

# Check webhook configuration
kubectl get validatingwebhookconfigurations.admissionregistration.k8s.io \
  -l gatekeeper.sh/system=yes

required-labels-template.yaml

apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredlabels
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredLabels
      validation:
        properties:
          labels:
            type: array
            items:
              type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredlabels

        violation[{"msg": msg}] {
          required := input.parameters.labels
          provided := input.review.object.metadata.labels
          missing := required[_]
          not provided[missing]
          msg := sprintf("Missing required label: %v", [missing])
        }

container-limits-template.yaml

apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8scontainerlimits
spec:
  crd:
    spec:
      names:
        kind: K8sContainerLimits
      validation:
        properties:
          cpu:
            type: string
          memory:
            type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8scontainerlimits

        missing_cpu_limits[container] {
          container := input.review.object.spec.containers[_]
          not container.resources.limits.cpu
        }

        missing_memory_limits[container] {
          container := input.review.object.spec.containers[_]
          not container.resources.limits.memory
        }

        violation[{"msg": msg}] {
          container := missing_cpu_limits[_]
          msg := sprintf("Container '%v' is missing CPU limits", [container.name])
        }

        violation[{"msg": msg}] {
          container := missing_memory_limits[_]
          msg := sprintf("Container '%v' is missing memory limits", [container.name])
        }

        violation[{"msg": msg}] {
          container := input.review.object.spec.containers[_]
          cpu_limit := container.resources.limits.cpu
          cpu_limit_num := to_number(trim_suffix(cpu_limit, "m"))
          cpu_limit_num > to_number(trim_suffix(input.parameters.cpu, "m"))
          msg := sprintf("Container '%v' CPU limit '%v' exceeds maximum '%v'", 
            [container.name, cpu_limit, input.parameters.cpu])
        }

require-labels-constraint.yaml

apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: must-have-environment-label
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment", "ReplicaSet"]
      - apiGroups: [""]
        kinds: ["Pod", "Service"]
    excludedNamespaces: ["kube-system", "gatekeeper-system"]
  parameters:
    labels: ["environment", "app", "version"]

container-limits-constraint.yaml

apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sContainerLimits
metadata:
  name: container-must-have-limits
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
      - apiGroups: [""]
        kinds: ["Pod"]
    excludedNamespaces: ["kube-system", "gatekeeper-system"]
  parameters:
    cpu: "1000m"
    memory: "2Gi"

Check Constraint Status

# List all constraints
kubectl get constraints

# Check specific constraint status
kubectl describe k8srequiredlabels must-have-environment-label

# View violations
kubectl get k8srequiredlabels must-have-environment-label -o yaml

# Check gatekeeper audit logs
kubectl logs -n gatekeeper-system -l control-plane=audit-controller

# Check controller manager logs
kubectl logs -n gatekeeper-system -l control-plane=controller-manager

Install Kyverno

# Install using kubectl
kubectl create -f https://github.com/kyverno/kyverno/releases/latest/download/install.yaml

# Or install using Helm
helm repo add kyverno https://kyverno.github.io/kyverno/
helm install kyverno kyverno/kyverno \
  --namespace kyverno \
  --create-namespace

# Verify installation
kubectl get pods -n kyverno

# Check webhook configuration
kubectl get validatingwebhookconfigurations \
  -l app.kubernetes.io/name=kyverno

require-labels-policy.yaml

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-labels
spec:
  validationFailureAction: enforce
  background: true
  rules:
  - name: check-environment-label
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Service
          - Deployment
    validate:
      message: "Label 'environment' is required"
      pattern:
        metadata:
          labels:
            environment: "?*"
  - name: check-app-label
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Service
          - Deployment
    validate:
      message: "Label 'app' is required"
      pattern:
        metadata:
          labels:
            app: "?*"

container-security-policy.yaml

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: container-security
spec:
  validationFailureAction: enforce
  background: true
  rules:
  - name: check-runAsNonRoot
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Deployment
    validate:
      message: "Containers must run as non-root"
      pattern:
        spec:
          =(securityContext):
            =(runAsNonRoot): true
          containers:
          - name: "*"
            =(securityContext):
              =(runAsNonRoot): true
  
  - name: require-resource-limits
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Deployment
    validate:
      message: "Containers must have resource limits"
      pattern:
        spec:
          containers:
          - name: "*"
            resources:
              limits:
                memory: "?*"
                cpu: "?*"
  
  - name: disallow-privileged
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Deployment
    validate:
      message: "Privileged containers are not allowed"
      pattern:
        spec:
          containers:
          - name: "*"
            =(securityContext):
              =(privileged): false

add-default-labels.yaml

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: add-default-labels
spec:
  rules:
  - name: add-managed-by-label
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Service
          - Deployment
    mutate:
      patchStrategicMerge:
        metadata:
          labels:
            +(managed-by): kyverno
            +(created-date): "{{ time_now_utc() }}"
  
  - name: add-security-context
    match:
      any:
      - resources:
          kinds:
          - Pod
          - Deployment
    mutate:
      patchStrategicMerge:
        spec:
          +(securityContext):
            +(runAsNonRoot): true
            +(fsGroup): 2000
          containers:
          - (name): "*"
            +(securityContext):
              +(allowPrivilegeEscalation): false
              +(capabilities):
                +(drop):
                - ALL
              +(readOnlyRootFilesystem): true
              +(runAsNonRoot): true

generate-network-policy.yaml

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: generate-default-network-policy
spec:
  rules:
  - name: default-deny-ingress
    match:
      any:
      - resources:
          kinds:
          - Namespace
    exclude:
      any:
      - resources:
          namespaces:
          - kube-system
          - kyverno
          - kube-public
    generate:
      apiVersion: networking.k8s.io/v1
      kind: NetworkPolicy
      name: default-deny-ingress
      namespace: "{{ request.object.metadata.name }}"
      data:
        spec:
          podSelector: {}
          policyTypes:
          - Ingress
  
  - name: generate-limit-range
    match:
      any:
      - resources:
          kinds:
          - Namespace
    exclude:
      any:
      - resources:
          namespaces:
          - kube-system
          - kyverno
    generate:
      apiVersion: v1
      kind: LimitRange
      name: default-limitrange
      namespace: "{{ request.object.metadata.name }}"
      data:
        spec:
          limits:
          - default:
              cpu: "500m"
              memory: "512Mi"
            defaultRequest:
              cpu: "100m"
              memory: "128Mi"
            type: Container

Check Kyverno Status

# List all policies
kubectl get clusterpolicies

# Check policy status
kubectl describe clusterpolicy require-labels

# View policy violations
kubectl get events --field-selector reason=PolicyViolation

# Check Kyverno logs
kubectl logs -n kyverno -l app.kubernetes.io/name=kyverno

# Get policy reports
kubectl get polr -A   # Policy Reports
kubectl get cpolr     # Cluster Policy Reports

# Check specific policy report
kubectl describe polr -n default

Install Polaris

# Install dashboard using kubectl
kubectl apply -f https://github.com/FairwindsOps/polaris/releases/latest/download/dashboard.yaml

# Or install using Helm
helm repo add fairwinds-stable https://charts.fairwinds.com/stable
helm install polaris fairwinds-stable/polaris \
  --namespace polaris \
  --create-namespace

# Install CLI tool
curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris_linux_amd64.tar.gz \
  | tar xz
sudo mv polaris /usr/local/bin/

# Install webhook (for admission control)
helm install polaris fairwinds-stable/polaris \
  --namespace polaris \
  --create-namespace \
  --set webhook.enable=true \
  --set dashboard.enable=false

polaris-config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: polaris-config
  namespace: polaris
data:
  config.yaml: |
    resources:
      cpuRequestsMissing: warning
      memoryRequestsMissing: warning
      cpuLimitsMissing: warning
      memoryLimitsMissing: warning
    images:
      tagNotSpecified: error
      pullPolicyNotAlways: warning
    healthChecks:
      readinessProbeNotSet: warning
      livenessProbeNotSet: warning
    networking:
      hostNetworkSet: error
      hostPortSet: warning
    security:
      hostIPCSet: error
      hostPIDSet: error
      notReadOnlyRootFilesystem: warning
      privilegeEscalationAllowed: error
      runningAsRoot: warning
      runningAsNonRoot: error
      readOnlyRootFilesystem: warning
      capabilities:
        error:
          ifAnyAdded:
            - SYS_ADMIN
            - NET_ADMIN
            - ALL
        warning:
          ifAnyAddedBeyond:
            - CHOWN
            - DAC_OVERRIDE
            - FSETID
            - FOWNER
            - MKNOD
            - NET_RAW
            - SETGID
            - SETUID
            - SETFCAP
            - SETPCAP
            - NET_BIND_SERVICE
            - SYS_CHROOT
            - KILL
            - AUDIT_WRITE
    exemptions:
      - controllerNames:
          - kube-apiserver
          - kube-proxy
          - kube-scheduler
          - etcd-manager-events
          - kube-controller-manager
          - kube-dns
          - etcd-manager-main
        rules:
          - hostNetworkSet
          - hostPortSet
          - hostIPCSet
          - hostPIDSet
          - runningAsRoot
      - controllerNames:
          - kube-flannel-ds
        rules:
          - notReadOnlyRootFilesystem
          - runningAsRoot
          - notRunningAsNonRoot
          - readOnlyRootFilesystem
          - capabilities

Polaris CLI Usage

# Audit entire cluster
polaris audit --output-format=pretty

# Audit specific namespace
polaris audit --output-format=json --namespace=default

# Audit from YAML files
polaris audit --audit-path=./k8s-manifests/

# Generate detailed report
polaris audit --output-format=json > polaris-audit-report.json

# Set minimum score threshold
polaris audit --set-exit-code-on-danger --set-exit-code-below-score=80

# Audit with custom config
polaris audit --config=custom-polaris-config.yaml

custom-checks-config.yaml

customChecks:
  requireEnvironmentLabel:
    successMessage: Environment label is set
    failureMessage: Environment label should be set
    category: Best Practices
    target: Container
    schema:
      '$schema': http://json-schema.org/draft-07/schema
      type: object
      properties:
        metadata:
          type: object
          properties:
            labels:
              type: object
              properties:
                environment:
                  type: string
              required:
                - environment
          required:
            - labels
      required:
        - metadata

  disallowLatestTag:
    successMessage: Image tag is not 'latest'
    failureMessage: Image tag should not be 'latest'
    category: Security
    target: Container
    schema:
      '$schema': http://json-schema.org/draft-07/schema
      type: object
      properties:
        image:
          type: string
          not:
            pattern: ':latest$|:latest@'
      required:
        - image

  requireSeccompProfile:
    successMessage: Seccomp profile is set
    failureMessage: Seccomp profile should be set
    category: Security
    target: Pod
    schema:
      '$schema': http://json-schema.org/draft-07/schema
      type: object
      properties:
        securityContext:
          type: object
          properties:
            seccompProfile:
              type: object
              properties:
                type:
                  type: string
                  enum: ["RuntimeDefault", "Localhost"]
              required:
                - type
          required:
            - seccompProfile
      required:
        - securityContext

multi-engine-strategy.yaml

# Use Case Distribution:

# 1. OPA Gatekeeper - Complex governance and compliance
#     - Regulatory compliance (SOX, HIPAA, PCI-DSS)
#     - Complex business logic
#     - External data integration
#     - Advanced RBAC policies

# 2. Kyverno - Day-to-day operations and mutations   
#     - Image policies and verification
#     - Resource mutations and generation
#     - Simple validation rules
#     - Developer-friendly policies

# 3. Polaris - Security scanning and best practices
#     - CI/CD security validation
#     - Best practice enforcement
#     - Developer education
#     - Periodic security audits

# Example namespace labeling strategy:
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    policy-engine: "gatekeeper"      # Use Gatekeeper for strict compliance
    security-scan: "polaris"        # Always scan with Polaris
    environment: "production"
  annotations:
    kyverno.io/exclude: "true"       # Exclude Kyverno mutations in prod
---
apiVersion: v1
kind: Namespace
metadata:
  name: development
  labels:
    policy-engine: "kyverno"         # Use Kyverno for flexibility
    security-scan: "polaris"        # Always scan with Polaris
    environment: "development"

policy-coordination.yaml

# Kyverno policy to add Gatekeeper exemptions
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: coordinate-with-gatekeeper
spec:
  rules:
  - name: add-gatekeeper-exemption
    match:
      any:
      - resources:
          kinds:
          - Pod
          namespaces:
          - development
          - testing
    mutate:
      patchStrategicMerge:
        metadata:
          annotations:
            +(gatekeeper.sh/exclude): "true"

---
# Gatekeeper constraint with Kyverno coordination
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: production-labels-only
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
    namespaces: ["production"]
    excludedNamespaces: []
  parameters:
    labels: ["environment", "app", "version", "team"]

---
# Polaris webhook configuration to exclude certain namespaces
apiVersion: v1
kind: ConfigMap
metadata:
  name: polaris-webhook-config
  namespace: polaris
data:
  config.yaml: |
    webhook:
      namespacesExcluded:
        - kube-system
        - polaris
        - gatekeeper-system
        - kyverno
      rules:
        # Only run security checks in webhook mode
        security:
          runningAsRoot: error
          privilegeEscalationAllowed: error
          capabilities:
            error:
              ifAnyAdded: ["SYS_ADMIN", "NET_ADMIN"]

.github/workflows/k8s-policy-validation.yml

name: Kubernetes Policy Validation

on:
  pull_request:
    paths:
      - 'k8s/**'
      - 'manifests/**'
      - 'charts/**'

jobs:
  policy-validation:
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Setup kubectl
      uses: azure/setup-kubectl@v3
      with:
        version: 'latest'
    
    - name: Install Polaris CLI
      run: |
        curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris_linux_amd64.tar.gz | tar xz
        sudo mv polaris /usr/local/bin/
    
    - name: Install Conftest (for OPA policies)
      run: |
        curl -L https://github.com/open-policy-agent/conftest/releases/latest/download/conftest_linux_x86_64.tar.gz | tar xz
        sudo mv conftest /usr/local/bin/
    
    - name: Install Kyverno CLI
      run: |
        curl -L https://github.com/kyverno/kyverno/releases/latest/download/kyverno-cli_linux_x86_64.tar.gz | tar xz
        sudo mv kyverno /usr/local/bin/
    
    - name: Validate with Polaris
      run: |
        echo "🔍 Running Polaris security validation..."
        polaris audit --audit-path=k8s/ --format=json > polaris-results.json
        
        # Check if there are high-severity issues
        high_issues=$(cat polaris-results.json | jq '.Results[] | select(.PodResult.ContainerResults[].Results[] | select(.Severity == "error")) | length' | wc -l)
        if [ $high_issues -gt 0 ]; then
          echo "❌ High-severity security issues found!"
          cat polaris-results.json | jq '.Results[] | select(.PodResult.ContainerResults[].Results[] | select(.Severity == "error"))'
          exit 1
        else
          echo "✅ Polaris validation passed!"
        fi
    
    - name: Validate with Conftest (OPA policies)
      run: |
        echo "🛡️ Running OPA policy validation..."
        if [ -d "policies/opa" ]; then
          find k8s/ -name "*.yaml" -o -name "*.yml" | xargs conftest verify --policy policies/opa/
        else
          echo "No OPA policies found, skipping..."
        fi
    
    - name: Validate with Kyverno CLI
      run: |
        echo "⚙️ Running Kyverno policy validation..."
        if [ -d "policies/kyverno" ]; then
          kyverno apply policies/kyverno/ --resource k8s/ --output json > kyverno-results.json
          
          # Check for policy violations
          violations=$(cat kyverno-results.json | jq '.results[] | select(.result == "fail") | length' | wc -l)
          if [ $violations -gt 0 ]; then
            echo "❌ Kyverno policy violations found!"
            cat kyverno-results.json | jq '.results[] | select(.result == "fail")'
            exit 1
          else
            echo "✅ Kyverno validation passed!"
          fi
        else
          echo "No Kyverno policies found, skipping..."
        fi
    
    - name: Generate Policy Report
      if: always()
      run: |
        echo "## 📊 Kubernetes Policy Validation Report" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        # Polaris results
        if [ -f polaris-results.json ]; then
          score=$(cat polaris-results.json | jq '.ClusterSummary.Score')
          echo "### Polaris Security Score: ${score}%" >> $GITHUB_STEP_SUMMARY
          
          errors=$(cat polaris-results.json | jq '.ClusterSummary.Results.Error')
          warnings=$(cat polaris-results.json | jq '.ClusterSummary.Results.Warning')
          successes=$(cat polaris-results.json | jq '.ClusterSummary.Results.Success')
          
          echo "- ❌ Errors: $errors" >> $GITHUB_STEP_SUMMARY
          echo "- ⚠️  Warnings: $warnings" >> $GITHUB_STEP_SUMMARY
          echo "- ✅ Successes: $successes" >> $GITHUB_STEP_SUMMARY
        fi
        
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "### Policy Engines Used" >> $GITHUB_STEP_SUMMARY
        echo "- 🔍 Polaris: Security best practices" >> $GITHUB_STEP_SUMMARY
        [ -d "policies/opa" ] && echo "- 🛡️ OPA: Custom governance policies" >> $GITHUB_STEP_SUMMARY
        [ -d "policies/kyverno" ] && echo "- ⚙️ Kyverno: Validation and mutation" >> $GITHUB_STEP_SUMMARY
    
    - name: Upload Policy Results
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: policy-validation-results
        path: |
          polaris-results.json
          kyverno-results.json

.pre-commit-config.yaml

repos:
- repo: https://github.com/FairwindsOps/polaris
  rev: 5.0.0
  hooks:
  - id: polaris-audit
    args: ['--audit-path', 'k8s/', '--set-exit-code-on-danger']

- repo: https://github.com/open-policy-agent/conftest
  rev: v0.46.0
  hooks:
  - id: conftest-verify
    args: ['--policy', 'policies/opa/']
    files: ^k8s/.*\.ya?ml$

- repo: local
  hooks:
  - id: kyverno-validate
    name: Kyverno Policy Validation
    entry: kyverno
    language: system
    args: ['apply', 'policies/kyverno/', '--resource']
    files: ^k8s/.*\.ya?ml$
    pass_filenames: true

policy-metrics-servicemonitor.yaml

# ServiceMonitor for Gatekeeper metrics
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: gatekeeper-metrics
  namespace: gatekeeper-system
spec:
  selector:
    matchLabels:
      control-plane: controller-manager
  endpoints:
  - port: metrics
    interval: 30s
    path: /metrics

---
# ServiceMonitor for Kyverno metrics   
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kyverno-metrics
  namespace: kyverno
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: kyverno
  endpoints:
  - port: metrics
    interval: 30s
    path: /metrics

---
# Custom policy violation exporter
apiVersion: apps/v1
kind: Deployment
metadata:
  name: policy-violation-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: policy-violation-exporter
  template:
    metadata:
      labels:
        app: policy-violation-exporter
    spec:
      containers:
      - name: exporter
        image: policy-exporter:latest
        ports:
        - containerPort: 8080
          name: metrics
        env:
        - name: PROMETHEUS_PORT
          value: "8080"
        - name: SCRAPE_INTERVAL
          value: "60s"

policy-dashboard.json

{
  "dashboard": {
    "title": "Kubernetes Policy Compliance",
    "panels": [
      {
        "title": "Policy Violations by Engine",
        "type": "stat",
        "targets": [
          {
            "expr": "sum by (engine) (policy_violations_total)",
            "legendFormat": "{{engine}}"
          }
        ]
      },
      {
        "title": "Gatekeeper Constraint Violations",
        "type": "table",
        "targets": [
          {
            "expr": "gatekeeper_violations{enforcement_action=\"deny\"}",
            "format": "table"
          }
        ]
      },
      {
        "title": "Kyverno Policy Results",
        "type": "piechart", 
        "targets": [
          {
            "expr": "sum by (result) (kyverno_policy_results_total)",
            "legendFormat": "{{result}}"
          }
        ]
      },
      {
        "title": "Security Score Trend",
        "type": "timeseries",
        "targets": [
          {
            "expr": "polaris_security_score",
            "legendFormat": "Security Score"
          }
        ]
      },
      {
        "title": "Top Violating Namespaces",
        "type": "bargauge",
        "targets": [
          {
            "expr": "topk(10, sum by (namespace) (policy_violations_total))",
            "legendFormat": "{{namespace}}"
          }
        ]
      }
    ]
  }
}

Kubernetes Policy Engine Mastery

You now have comprehensive knowledge of Kubernetes policy engines including: