v0.2.3 - Identity rules, test harness, CVE decoupling and Learn portal

.github/workflows/ci.yml

@@ -328,17 +328,27 @@ jobs:
               print(f"All compliance controls map to existing rule files. ({len(existing_ids)} rules checked)")
           PYEOF
 
+      # ── CHECK 8: Rule regression tests (MockAzureClient, no Azure creds) ──
+      - name: Rule regression tests
+        id: rule_tests
+        env:
+          DATABASE_URL: "postgresql://ci:ci@localhost/ci_db"
+        run: |
+          echo "=== Running rule regression tests ==="
+          pytest tests/test_rules_*.py -v --tb=short
+
       # ── Final summary — always runs, shows per-check pass/fail ────────
       - name: CI Summary
         if: always()
         env:
-          SYNTAX:    ${{ steps.syntax_check.outcome }}
-          STRUCTURE: ${{ steps.structure_check.outcome }}
-          CREDS:     ${{ steps.cred_scan.outcome }}
-          PLAYBOOK:  ${{ steps.playbook_check.outcome }}
-          JSON:      ${{ steps.json_check.outcome }}
-          API:       ${{ steps.api_check.outcome }}
-          XREF:      ${{ steps.xref_check.outcome }}
+          SYNTAX:     ${{ steps.syntax_check.outcome }}
+          STRUCTURE:  ${{ steps.structure_check.outcome }}
+          CREDS:      ${{ steps.cred_scan.outcome }}
+          PLAYBOOK:   ${{ steps.playbook_check.outcome }}
+          JSON:       ${{ steps.json_check.outcome }}
+          API:        ${{ steps.api_check.outcome }}
+          XREF:       ${{ steps.xref_check.outcome }}
+          RULE_TESTS: ${{ steps.rule_tests.outcome }}
         run: |
           python - <<'PYEOF'
           import os
@@ -351,6 +361,7 @@ jobs:
               ("Compliance JSON validation",           os.environ["JSON"]),
               ("API syntax check",                     os.environ["API"]),
               ("Compliance vs rule cross-reference",   os.environ["XREF"]),
+              ("Rule regression tests",                os.environ["RULE_TESTS"]),
           ]
 
           labels = {

README.md

@@ -243,6 +243,16 @@ MIT — free to use, modify, and distribute.
 
 ## Learn OpenShield
 
+Learn OpenShield covers:
+
+- Azure CSPM fundamentals
+- OpenShield architecture
+- Compliance mappings
+- Remediation workflows
+- Contributor onboarding
+- Documentation navigation
+
+Live Learning Portal: https://openshieldlearn.netlify.app/learn/
 Full documentation, the security rules gallery, blog, and interactive playground are available at the project website:
 
 **[openshield-website.vercel.app](https://openshield-website.vercel.app)**

api/models/finding.py

@@ -136,7 +136,8 @@ def create_tables(self) -> None:
                     started_at      TIMESTAMPTZ NOT NULL,
                     completed_at    TIMESTAMPTZ,
                     total_findings  INTEGER DEFAULT 0,
-                    score           INTEGER DEFAULT NULL
+                    score           INTEGER DEFAULT NULL,
+                    cve_enrichment_status TEXT DEFAULT 'PENDING'
                 );
             """)
             cur.execute("""
@@ -203,6 +204,10 @@ def run_migrations(self) -> None:
                         ADD COLUMN IF NOT EXISTS cvss_score        FLOAT   DEFAULT NULL,
                         ADD COLUMN IF NOT EXISTS exploit_available BOOLEAN DEFAULT FALSE
                 """)
+                cur.execute("""
+                    ALTER TABLE scans
+                        ADD COLUMN IF NOT EXISTS cve_enrichment_status TEXT DEFAULT 'PENDING'
+                """)
             conn.commit()
             logger.info("CVE migrations applied successfully")
         except Exception as e:
@@ -219,8 +224,8 @@ def save_scan(self, scan_result: Dict[str, Any]) -> None:
         with conn.cursor() as cur:
             cur.execute(
                 """
-                INSERT INTO scans (scan_id, subscription_id, started_at, completed_at, total_findings, score)
-                VALUES (%s, %s, %s, %s, %s, %s)
+                INSERT INTO scans (scan_id, subscription_id, started_at, completed_at, total_findings, score, cve_enrichment_status)
+                VALUES (%s, %s, %s, %s, %s, %s, %s)
                 ON CONFLICT (scan_id) DO NOTHING
                 """,
                 (
@@ -230,6 +235,7 @@ def save_scan(self, scan_result: Dict[str, Any]) -> None:
                     scan_result["completed_at"],
                     scan_result["total_findings"],
                     scan_result.get("score"),
+                    scan_result.get("cve_enrichment_status", "PENDING"),
                 ),
             )
             for f in scan_result.get("findings", []):
@@ -345,6 +351,17 @@ def update_cve_fields(self, findings: List[Dict[str, Any]]) -> None:
                 )
         conn.commit()
 
+    def update_scan_enrichment_status(self, scan_id: str, status: str) -> None:
+        """Update the CVE enrichment status for a specific scan."""
+        conn = self._get_conn()
+        with conn.cursor() as cur:
+            cur.execute(
+                "UPDATE scans SET cve_enrichment_status = %s WHERE scan_id = %s",
+                (status, scan_id),
+            )
+        conn.commit()
+        logger.info("Updated scan %s enrichment status to %s", scan_id, status)
+
     def get_scans(self) -> List[Dict[str, Any]]:
         """Return all scan records ordered by most recent first."""
         conn = self._get_conn()
@@ -387,21 +404,25 @@ def get_cve_summary(self) -> Dict[str, Any]:
         conn = self._get_conn()
         with conn.cursor() as cur:
             cur.execute("""
-                SELECT
-                    COUNT(*) as total_findings,
-                    COUNT(CASE WHEN exploit_available = TRUE THEN 1 END) as exploit_count,
-                    MAX(cvss_score) as max_cvss_score,
-                    AVG(cvss_score) as avg_cvss_score,
-                    COUNT(CASE WHEN cvss_score >= 9.0 THEN 1 END) as critical_cve_count
-                FROM findings
-                WHERE scan_id = (
+                SELECT 
+                    s.cve_enrichment_status,
+                    COUNT(f.*) as total_findings,
+                    COUNT(CASE WHEN f.exploit_available = TRUE THEN 1 END) as exploit_count,
+                    MAX(f.cvss_score) as max_cvss_score,
+                    AVG(f.cvss_score) as avg_cvss_score,
+                    COUNT(CASE WHEN f.cvss_score >= 9.0 THEN 1 END) as critical_cve_count
+                FROM scans s
+                LEFT JOIN findings f ON s.scan_id = f.scan_id
+                WHERE s.scan_id = (
                     SELECT scan_id FROM scans WHERE total_findings > 0 ORDER BY started_at DESC LIMIT 1
                 )
+                GROUP BY s.cve_enrichment_status
             """)
             row = cur.fetchone()
 
         if not row:
             return {
+                "status": "UNKNOWN",
                 "total_findings": 0,
                 "exploit_count": 0,
                 "max_cvss_score": None,
@@ -410,11 +431,12 @@ def get_cve_summary(self) -> Dict[str, Any]:
             }
 
         return {
-            "total_findings": row[0],
-            "exploit_count": row[1],
-            "max_cvss_score": row[2],
-            "avg_cvss_score": round(row[3], 2) if row[3] is not None else None,
-            "critical_cve_count": row[4],
+            "status": row[0],
+            "total_findings": row[1],
+            "exploit_count": row[2],
+            "max_cvss_score": row[3],
+            "avg_cvss_score": round(row[4], 2) if row[4] is not None else None,
+            "critical_cve_count": row[5],
         }
 
     def get_compliance_score(self, framework: str) -> Dict[str, Any]:

api/routes/findings.py

@@ -6,7 +6,8 @@
 from flask import Blueprint, g, jsonify, request
 
 from api.models.finding import DatabaseManager
-from scanner.cve_correlator import enrich_findings
+
+_PLAYBOOKS_DIR = Path(__file__).parent.parent.parent / "playbooks" / "cli"
 
 _PLAYBOOKS_DIR = Path(__file__).parent.parent.parent / "playbooks" / "cli"
 
@@ -39,16 +40,6 @@ def list_findings():
         }
         db = _get_db()
         findings = db.get_findings(filters)
-        legacy_findings = [
-            f
-            for f in findings
-            if f.get("cve_references") is None
-            and f.get("cvss_score") is None
-            and f.get("exploit_available") is None
-        ]
-        if legacy_findings:
-            enrich_findings(legacy_findings)
-            db.update_cve_fields(legacy_findings)
         return jsonify({"count": len(findings), "findings": findings})
     except Exception as exc:
         logger.error("Failed to list findings: %s", exc)

api/routes/scans.py

@@ -5,6 +5,7 @@
 from flask import Blueprint, g, jsonify, request
 
 from api.models.finding import DatabaseManager
+from scanner.cve_correlator import enrich_findings
 
 scans_bp = Blueprint("scans", __name__)
 logger = logging.getLogger(__name__)
@@ -79,4 +80,50 @@ def trigger_scan():
 
     except Exception as exc:
         logger.error("Critical error in trigger_scan route: %s", exc, exc_info=True)
-        return jsonify({"error": "Critical route failure", "detail": str(exc)}), 500
+        return jsonify({"error": "Critical route failure", "detail": str(exc)}), 500
+
+
+@scans_bp.post("/api/scans/<scan_id>/enrich")
+def enrich_scan(scan_id):
+    """Trigger CVE enrichment for an existing scan."""
+    try:
+        db = _get_db()
+
+        # Check current status to avoid redundant NVD calls
+        scans = db.get_scans()
+        current_scan = next((s for s in scans if str(s["scan_id"]) == scan_id), None)
+
+        if not current_scan:
+            return jsonify({"error": "Scan not found"}), 404
+
+        status = current_scan.get("cve_enrichment_status")
+        if status == "COMPLETED":
+            return jsonify({"message": "Scan already enriched", "scan_id": scan_id}), 200
+        if status == "ENRICHING":
+            return jsonify({"message": "Enrichment already in progress", "scan_id": scan_id}), 202
+
+        findings = db.get_findings({"scan_id": scan_id})
+        if not findings:
+            return jsonify({"error": "No findings found for this scan"}), 404
+
+        logger.info("Enriching %d findings for scan %s", len(findings), scan_id)
+        db.update_scan_enrichment_status(scan_id, "ENRICHING")
+
+        try:
+            enriched = enrich_findings(findings)
+            db.update_cve_fields(enriched)
+            db.update_scan_enrichment_status(scan_id, "COMPLETED")
+        except Exception as exc:
+            logger.error("Enrichment failed for scan %s: %s", scan_id, exc)
+            db.update_scan_enrichment_status(scan_id, "FAILED")
+            return jsonify({"error": "Enrichment failed", "detail": str(exc)}), 500
+
+        return jsonify({
+            "scan_id": scan_id,
+            "status": "COMPLETED",
+            "enriched_count": len(enriched)
+        })
+
+    except Exception as exc:
+        logger.error("Failed to enrich scan %s: %s", scan_id, exc)
+        return jsonify({"error": "Internal server error", "detail": str(exc)}), 500

compliance/frameworks/cis_azure_benchmark.json

@@ -77,6 +77,31 @@
       "control_id": "1.15",
       "control_name": "Ensure that 'Guest invite restrictions' is set to 'Only users assigned to specific admin roles can invite guest users'",
       "description": "Unrestricted guest user invitation settings allow any member of the organisation to invite external users into the tenant without administrative review. This bypasses centralised approval for external identity provisioning and increases the risk of unauthorised access by untrusted parties."
+    },
+        "AZ-IDN-005": {
+      "control_id": "1.3",
+      "control_name": "Ensure guest users are reviewed on a monthly basis",
+      "description": "Guest accounts assigned to high privilege roles in Entra ID allow external identities to perform administrative actions in the tenant. CIS 1.3 requires that guest users are reviewed and that privileged access is restricted to internal accounts only. Any guest user holding a role such as Global Administrator, Security Administrator, or User Administrator must have that assignment removed immediately."
+    },
+    "AZ-IDN-006": {
+      "control_id": "1.14",
+      "control_name": "Ensure that service principal passwords are rotated within 90 days",
+      "description": "Service principal client secrets older than 90 days or with no expiry date represent a persistent credential risk. CIS 1.14 requires that service principal passwords are rotated at least every 90 days. Secrets that never expire remain valid indefinitely if leaked, giving an attacker permanent access to the application and its Azure permissions."
+    },
+    "AZ-IDN-007": {
+      "control_id": "1.1",
+      "control_name": "Ensure that multi-factor authentication is enabled for all privileged users",
+      "description": "Active users in Entra ID with no MFA methods registered are vulnerable to password-based attacks including spray and phishing. CIS 1.1 requires that MFA is enabled for all users, particularly those with privileged access. Users without MFA registered must be required to enrol before they can access Azure resources."
+    },
+    "AZ-IDN-008": {
+      "control_id": "1.23",
+      "control_name": "Ensure that custom subscription roles do not exist",
+      "description": "Custom RBAC roles with wildcard actions (*) at subscription scope grant Owner-equivalent permissions and violate the principle of least privilege. CIS 1.23 requires that custom subscription roles do not have wildcard permissions. These roles must be replaced with definitions that specify only the exact actions required for the intended use case."
+    },
+    "AZ-IDN-009": {
+      "control_id": "5.2.1",
+      "control_name": "Ensure that activity log alert exists for Create Policy Assignment",
+      "description": "A subscription without an activity log alert for role assignment changes cannot detect privilege escalation in real time. CIS 5.2.1 requires that activity log alerts exist for administrative operations including role assignment writes. Without this alert, an attacker who elevates their own permissions will go undetected until the next manual review."
     },
     "AZ-DB-001": {
       "control_id": "4.3.1",

compliance/frameworks/iso27001.json

@@ -77,6 +77,31 @@
       "control_id": "A.9.2.1",
       "control_name": "User registration and de-registration",
       "description": "Unrestricted guest user invitations allow any organisation member to register external identities into the tenant without centralised review or approval. A.9.2.1 requires that users and external parties should be registered before access."
+    },
+        "AZ-IDN-005": {
+      "control_id": "A.9.2.3",
+      "control_name": "Management of privileged access rights",
+      "description": "The allocation and use of privileged access rights must be restricted and controlled. Guest accounts in Entra ID with high privilege roles represent uncontrolled privileged access by external identities. A.9.2.3 requires that the allocation of privileged access rights is controlled through a formal authorisation process and that privileged roles are assigned only to internal accounts with a verified business need."
+    },
+    "AZ-IDN-006": {
+      "control_id": "A.9.4.3",
+      "control_name": "Password management system",
+      "description": "Service principal client secrets with no expiry or older than 90 days violate password management controls. A.9.4.3 requires that password management systems enforce quality and lifecycle requirements including regular rotation. Non-expiring secrets must have an expiry date set and secrets older than 90 days must be rotated immediately."
+    },
+    "AZ-IDN-007": {
+      "control_id": "A.9.4.2",
+      "control_name": "Secure log-on procedures",
+      "description": "Users without MFA registered in Entra ID authenticate with a single factor, which does not meet secure log-on requirements. A.9.4.2 requires that access to systems and applications is controlled by a secure log-on procedure. Multi-factor authentication must be required for all active user accounts to prevent unauthorised access through compromised passwords."
+    },
+    "AZ-IDN-008": {
+      "control_id": "A.9.2.3",
+      "control_name": "Management of privileged access rights",
+      "description": "Custom RBAC roles with wildcard permissions at subscription scope are a form of uncontrolled privileged access that is harder to audit than built-in roles. A.9.2.3 requires that privileged access rights are allocated only through a formal authorisation process and are regularly reviewed. Custom roles with wildcard actions must be narrowed to specific required permissions or removed if unused."
+    },
+    "AZ-IDN-009": {
+      "control_id": "A.12.4.1",
+      "control_name": "Event logging",
+      "description": "Subscriptions without an activity log alert for role assignment changes fail to generate actionable security events when privileged access is granted. A.12.4.1 requires that event logs recording user activities and security-relevant events are produced and maintained. An activity log alert for Microsoft.Authorization/roleAssignments/write must be configured and routed to a monitored channel."
     },
     "AZ-DB-001": {
       "control_id": "A.13.1.1",