Optimize behat tests

stepup/tests/behat/.gitignore

@@ -1 +1,2 @@
-/vendor
+/vendor
+setup.sql

stepup/tests/behat/config/behat.yml

@@ -22,6 +22,7 @@ default:
             raUrl: 'https://ra.dev.openconext.local'
         - ApiFeatureContext:
             apiUrl: 'https://middleware.dev.openconext.local'
+        - ReplayContext: ~
         - Behat\MinkExtension\Context\MinkContext
 
   extensions:

stepup/tests/behat/features/bootstrap/FeatureContext.php

@@ -4,6 +4,7 @@
 use Behat\Behat\Hook\Scope\BeforeScenarioScope;
 use Behat\MinkExtension\Context\MinkContext;
 use Behat\Behat\Hook\Scope\BeforeFeatureScope;
+use Behat\Testwork\Hook\Scope\BeforeSuiteScope;
 use Ramsey\Uuid\Uuid;
 use Surfnet\StepupBehat\Factory\CommandPayloadFactory;
 use Surfnet\StepupBehat\Repository\SecondFactorRepository;
@@ -44,24 +45,64 @@ class FeatureContext implements Context
      */
     private $institutionConfiguration;
 
+    public static function execCommand(string $command): void
+    {
+        $output = [];
+        $returnCode = -1;
+        $result = exec($command, $output, $returnCode);
+
+        if($result === false) {
+            echo "Failed executing command\n";
+            die();
+        }
+
+        foreach ($output as $line) {
+            echo $line."\n";
+        }
+
+        if ($returnCode !== 0) {
+            die();
+        }
+    }
+
     /**
-     * @BeforeFeature
+     * @BeforeSuite
      */
-    public static function setupDatabase(BeforeFeatureScope $scope)
+    public static function setupDatabase(BeforeSuiteScope $scope)
     {
         // Generate test databases
         echo "Preparing test schemas\n";
-        shell_exec("docker exec -t stepup-middleware-1 bin/console doctrine:schema:drop --env=smoketest --force");
-        shell_exec("docker exec -t stepup-gateway-1  bin/console doctrine:schema:drop --env=smoketest --force");
-        shell_exec("docker exec -t stepup-middleware-1 bin/console doctrine:schema:create --env=smoketest");
-        shell_exec("docker exec -t stepup-gateway-1 bin/console doctrine:schema:create --env=smoketest");
+        self::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:drop --em=middleware --env=smoketest --force');
+        self::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:drop --em=gateway --env=smoketest --force');
+        self::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:create --em=middleware --env=smoketest');
+        self::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:create --em=gateway --env=smoketest');
 
-        echo "Replaying event stream\n";
         // Import the events.sql into middleware
-        shell_exec("mysql -uroot -psecret middleware_test -h mariadb < ./fixtures/events.sql");
-        shell_exec("./fixtures/middleware-push-config.sh");
+        echo "Add events to test database\n";
+        self::execCommand("mysql -uroot -psecret middleware_test -h mariadb < ./fixtures/events.sql");
+
         // Perform an event replay
-        shell_exec("docker exec -t stepup-middleware-1 bin/console middleware:event:replay --env=smoketest_event_replay --no-interaction -q");
+        echo "Replaying event stream\n";
+        self::execCommand("docker exec -t stepup-middleware-1 bin/console middleware:event:replay --env=smoketest_event_replay --no-interaction -vvv");
+
+        // Push config
+        echo "Push Middleware config\n";
+        self::execCommand("./fixtures/middleware-push-config.sh");
+        self::execCommand("./fixtures/middleware-push-whitelist.sh");
+        self::execCommand("./fixtures/middleware-push-institution.sh");
+
+        // Write base setup for initializing features
+        echo "Dump empty setup to mysql file\n";
+        self::execCommand("mysqldump -h mariadb -u root -psecret --single-transaction --databases middleware_test gateway_test > setup.sql");
+    }
+
+    /**
+     * @BeforeFeature
+     */
+    public static function load(BeforeFeatureScope $scope)
+    {
+        // restore base setup
+        self::execCommand("mysql -h mariadb -u root -psecret < setup.sql");
     }
 
     /**

stepup/tests/behat/features/bootstrap/ReplayContext.php

@@ -0,0 +1,44 @@
+<?php
+
+use Behat\Behat\Context\Context;
+use Behat\Behat\Hook\Scope\BeforeScenarioScope;
+use Behat\MinkExtension\Context\MinkContext;
+
+class ReplayContext implements Context
+{
+    /**
+     * @var \Behat\MinkExtension\Context\MinkContext
+     */
+    private $minkContext;
+
+    /**
+     * @BeforeScenario
+     */
+    public function gatherContexts(BeforeScenarioScope $scope)
+    {
+        $environment = $scope->getEnvironment();
+
+        $this->minkContext = $environment->getContext(MinkContext::class);
+    }
+
+    /**
+     * @Given a replay is performed
+     */
+    public function replay()
+    {
+        // Generate test databases
+        echo "Preparing test schemas\n";
+        FeatureContext::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:drop --em=middleware --env=smoketest --force');
+        FeatureContext::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:drop --em=gateway --env=smoketest --force');
+        FeatureContext::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:create --em=middleware --env=smoketest');
+        FeatureContext::execCommand('docker exec -t stepup-middleware-1 bin/console doctrine:schema:create --em=gateway --env=smoketest');
+
+        // Import the events.sql into middleware
+        echo "Add events to test database\n";
+        FeatureContext::execCommand("mysql -uroot -psecret middleware_test -h mariadb < ./fixtures/eventstream.sql");
+
+        // Perform an event replay
+        echo "Replaying event stream\n";
+        FeatureContext::execCommand("docker exec -t stepup-middleware-1 bin/console middleware:event:replay --env=smoketest_event_replay --no-interaction -vvv");
+    }
+}

stepup/tests/behat/features/bootstrap/SecondFactorAuthContext.php

@@ -387,14 +387,6 @@ public function authenticateWithIdentityProviderForWithStepup($userName)
         $this->minkContext->pressButton('Yes, continue');
     }
 
-    private function passTroughIdentityProviderAssertionConsumerService()
-    {
-        $this->minkContext->assertPageAddress('https://gateway.dev.openconext.local/authentication/consume-assertion');
-
-        $this->minkContext->assertPageNotContainsText('Incorrect username or password');
-        $this->minkContext->pressButton('Submit');
-    }
-
     /**
      * @Then I am logged on the service provider
      */

stepup/tests/behat/features/bootstrap/SelfServiceContext.php

@@ -508,7 +508,7 @@ private function getLastSentEmail()
         }
 
         $messages = json_decode($response);
-        if (!$messages) {
+        if (!is_array($messages)) {
             throw new Exception(
                 'Unable to parse mailcatcher response'
             );

stepup/tests/behat/features/identity.feature

@@ -3,6 +3,11 @@ Feature: A (S)RA(A) user reads identities of StepUp users in the middleware API
   As a (S)RA(A) user
   I must be able to read from the middleware API
 
+  Scenario: Provision the following users:
+    Given a user "jane-a1" identified by "urn:collab:person:institution-a.example.com:jane-a-ra" from institution "institution-a.example.com" with UUID "00000000-0000-4000-8000-000000000001"
+    And a user "joe-a1" identified by "urn:collab:person:institution-a.example.com:jane-a-ra" from institution "institution-a.example.com" with UUID "00000000-0000-4000-8000-000000000002"
+    And a user "jill-a1" identified by "urn:collab:person:institution-a.example.com:jane-a-ra" from institution "institution-a.example.com" with UUID "00000000-0000-4000-8000-000000000003"
+
   Scenario: A (S)RA(A) user reads identities without additional authorization context
     Given I authenticate with user "ra" and password "secret"
     When I request "GET /identity?institution=institution-a.example.com"

stepup/tests/behat/features/ra_candidate.feature

@@ -23,11 +23,7 @@ Feature: A RAA manages ra candidates in the ra environment
     Then I should see the following candidates:
       | name                                  | institution               |
       | jane-a-ra                             | institution-a.example.com |
-      | jane-b1 institution-b.example.com     | institution-b.example.com |
-      | user-b-ra institution-b.example.com   | institution-b.example.com |
-      | user-b5 institution-b.example.com     | institution-b.example.com |
       | Admin                                 | dev.openconext.local      |
-      | SRAA2                                 | dev.openconext.local      |
 
   Scenario: SRAA user checks if "Jane Toppan" is a candidate for all institutions (with filtering on institution-a)
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
@@ -42,9 +38,6 @@ Feature: A RAA manages ra candidates in the ra environment
     When I visit the RA promotion page
     Then I should see the following candidates for "institution-b.example.com":
       | name                                | institution               |
-      | jane-b1 institution-b.example.com   | institution-b.example.com |
-      | user-b-ra institution-b.example.com | institution-b.example.com |
-      | user-b5 institution-b.example.com   | institution-b.example.com |
 
   Scenario: SRAA user demotes "jane-a-ra" to no longer be an RAA for "institution-a"
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
@@ -57,18 +50,12 @@ Feature: A RAA manages ra candidates in the ra environment
     Then I should see the following candidates for "institution-a.example.com":
       | name                                | institution               |
       | jane-a-ra                           | institution-a.example.com |
-      | jane-b1 institution-b.example.com   | institution-b.example.com |
-      | user-b-ra institution-b.example.com | institution-b.example.com |
-      | user-b5 institution-b.example.com   | institution-b.example.com |
 
   Scenario: SRAA user checks if "Jane Toppan" is not a candidate for "institution-b"
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
     When I visit the RA promotion page
     Then I should see the following candidates for "institution-b.example.com":
       | name                                | institution               |
-      | jane-b1 institution-b.example.com   | institution-b.example.com |
-      | user-b-ra institution-b.example.com | institution-b.example.com |
-      | user-b5 institution-b.example.com   | institution-b.example.com |
 
   Scenario: SRAA user checks if "Jane Toppan" is not listed for "institution-a"
     Given I am logged in into the ra portal as "admin" with a "yubikey" token

stepup/tests/behat/features/ra_multiple_tokens.feature

@@ -54,11 +54,7 @@ Feature: A RAA (jane a ra) has two loa 3 tokens which makes her a valid RA candi
     Then I should see the following candidates:
       | name                                | institution               |
       | jane-a-ra                           | institution-a.example.com |
-      | jane-b1 institution-b.example.com   | institution-b.example.com |
-      | user-b5 institution-b.example.com   | institution-b.example.com |
-      | user-b-ra institution-b.example.com | institution-b.example.com |
       | Admin                               | dev.openconext.local      |
-      | SRAA2                               | dev.openconext.local      |
 
   Scenario: SRAA user checks if "jane-a-ra" is a candidate for institutions if relieved from the RAA role
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
@@ -68,11 +64,7 @@ Feature: A RAA (jane a ra) has two loa 3 tokens which makes her a valid RA candi
     And I should see the following candidates:
       | name                                | institution               |
       | jane-a-ra                           | institution-a.example.com |
-      | jane-b1 institution-b.example.com   | institution-b.example.com |
-      | user-b5 institution-b.example.com   | institution-b.example.com |
-      | user-b-ra institution-b.example.com | institution-b.example.com |
       | Admin                               | dev.openconext.local      |
-      | SRAA2                               | dev.openconext.local      |
 
   Scenario: Sraa revokes only one vetted token from "jane-a-ra" and that shouldn't remove her as candidate
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
@@ -82,11 +74,7 @@ Feature: A RAA (jane a ra) has two loa 3 tokens which makes her a valid RA candi
       And I should see the following candidates:
         | name                                | institution               |
         | jane-a-ra                           | institution-a.example.com |
-        | jane-b1 institution-b.example.com   | institution-b.example.com |
-        | user-b5 institution-b.example.com   | institution-b.example.com |
-        | user-b-ra institution-b.example.com | institution-b.example.com |
         | Admin                               | dev.openconext.local      |
-        | SRAA2                               | dev.openconext.local      |
 
   Scenario: Sraa revokes the last vetted token from "Jane Toppan" and that must remove her as candidate
     Given I am logged in into the ra portal as "admin" with a "yubikey" token
@@ -95,8 +83,4 @@ Feature: A RAA (jane a ra) has two loa 3 tokens which makes her a valid RA candi
     Then I visit the RA promotion page
       And I should see the following candidates:
         | name                                | institution               |
-        | jane-b1 institution-b.example.com   | institution-b.example.com |
-        | user-b5 institution-b.example.com   | institution-b.example.com |
-        | user-b-ra institution-b.example.com | institution-b.example.com |
         | Admin                               | dev.openconext.local      |
-        | SRAA2                               | dev.openconext.local      |

stepup/tests/behat/features/replay.feature

@@ -0,0 +1,11 @@
+Feature: A replay is performed on Middleware
+  In order to replay an event stream
+  On the command line
+  I expect the last event to be reflected in the data set
+
+  Scenario: After a replay is performed I would expect the last event reflected in the data set
+    Given a replay is performed
+    Given I authenticate with user "ra" and password "secret"
+      And I request "GET /identity?institution=institution-b.example.com&NameID=urn:collab:person:institution-b.example.com:joe-b5"
+    Then the api response status code should be 200
+      And the "items" property should contain 1 items