From 8d9f4efe1015eaa28561b14a84bfec172a88833c Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 16:52:09 +0900 Subject: [PATCH 01/10] Start work on adding Appwrite Tests --- .github/workflows/tests.yml | 16 +- Dockerfile | 30 +- composer.json | 3 +- docker-compose.yml | 768 +++++++++++++++++- .../{Mock.php => MockDestination.php} | 6 +- tests/Migration/E2E/Sources/AppwriteTest.php | 143 ++++ tests/Migration/E2E/Sources/NHostTest.php | 12 +- tests/Migration/E2E/Sources/SupabaseTest.php | 12 +- 8 files changed, 933 insertions(+), 57 deletions(-) rename tests/Migration/E2E/Adapters/{Mock.php => MockDestination.php} (93%) create mode 100644 tests/Migration/E2E/Sources/AppwriteTest.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b51d5e71..19069cce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,16 +4,22 @@ on: push: { branches: [main] } jobs: - tests: - name: Run Test Suite + source_e2e_tests: + name: Run Source E2E Tests runs-on: ubuntu-latest env: COMPOSE_FILE: docker-compose.yml - + strategy: + fail-fast: false + matrix: + adapter: [Appwrite, Supabase, NHost] steps: - name: Checkout uses: actions/checkout@v2 - - name: Run Tests + - name: Bring up services + run: docker-compose up -d --build + + - name: Run ${{matrix.adapter}} Tests run: | - docker-compose up -d --build && sleep 5 && docker compose exec tests php ./vendor/bin/phpunit \ No newline at end of file + docker compose exec tests php ./vendor/bin/phpunit /usr/src/code/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f62c0263..1a23bb9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,39 @@ FROM supabase/postgres:15.1.0.96 as supabase-db -COPY tests/Migration/resources/supabase/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql -COPY tests/Migration/resources/supabase/2_main.sql /docker-entrypoint-initdb.d/2_main.sql +COPY tests/Migration/Resources/supabase/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql +COPY tests/Migration/Resources/supabase/2_main.sql /docker-entrypoint-initdb.d/2_main.sql RUN rm -rf /docker-entrypoint-initdb.d/migrate.sh FROM postgres:alpine3.18 as nhost-db -COPY tests/Migration/resources/nhost/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql -COPY tests/Migration/resources/nhost/2_main.sql /docker-entrypoint-initdb.d/2_main.sql +COPY tests/Migration/Resources/nhost/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql +COPY tests/Migration/Resources/nhost/2_main.sql /docker-entrypoint-initdb.d/2_main.sql FROM composer:2.0 as composer + WORKDIR /usr/local/src/ COPY composer.json /usr/local/src/ RUN composer install --ignore-platform-reqs -FROM php:8.1.21-fpm-alpine3.18 as tests +FROM php:8.1.21 as tests + +# Install Appwrite Toolkit +RUN set -ex \ + && mkdir -p /app/toolkit \ + && cd /app/toolkit \ + && curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \ + && bash nodesource_setup.sh \ + && apt-get install nodejs git -y \ + && git clone https://github.com/Meldiron/appwrite-toolkit.git \ + && cd appwrite-toolkit \ + && git checkout feat-refactor-cli \ + && npm install \ + && npm link + # Postgres RUN set -ex \ - && apk --no-cache add postgresql-libs postgresql-dev \ + && apt-get install -y libpq-dev \ && docker-php-ext-install pdo pdo_pgsql \ - && apk del postgresql-dev + && apt-get remove -y libpq-dev + COPY ./src /app/src COPY ./tests /app/src/tests COPY --from=composer /usr/local/src/vendor /app/vendor diff --git a/composer.json b/composer.json index 2ea999ee..ccc46484 100644 --- a/composer.json +++ b/composer.json @@ -24,6 +24,7 @@ "require-dev": { "phpunit/phpunit": "9.*", "vlucas/phpdotenv": "5.*", - "laravel/pint": "1.*" + "laravel/pint": "1.*", + "utopia-php/cli": "^0.18.0" } } diff --git a/docker-compose.yml b/docker-compose.yml index a1512322..441b5a44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,25 @@ -version: '3' - services: + ## Tests + tests: + build: + context: . + target: tests + networks: + - tests + volumes: + - ./tests:/app/tests + - ./src:/app/src + - ./phpunit.xml:/app/phpunit.xml + working_dir: /app + depends_on: + - supabase-db + - nhost-db + - nhost-storage + environment: + - NHOST_DB_URL=postgres://postgres:postgres@nhost-db:5432/postgres + - SUPABASE_DB_URL=postgres://postgres:postgres@supabase-db:5432/postgres + + ## Supabase supabase-db: build: context: . @@ -14,6 +33,15 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + supabase-api: + image: mockoon/cli:latest + command: ["--data", "/mockoon/api.json", "--port", "80", "--disable-log-to-file"] + networks: + - tests + volumes: + - ./tests/Migration/resources/supabase:/mockoon + + ## NHost nhost-db: build: context: . @@ -34,33 +62,727 @@ services: - tests volumes: - ./tests/Migration/resources/nhost:/mockoon - - supabase-api: - image: mockoon/cli:latest - command: ["--data", "/mockoon/api.json", "--port", "80", "--disable-log-to-file"] + + ## Appwrite + appwrite: + image: appwrite/appwrite:1.5.7 + container_name: appwrite + restart: unless-stopped networks: - tests - volumes: - - ./tests/Migration/resources/supabase:/mockoon + - appwrite + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_LOCALE + - _APP_CONSOLE_WHITELIST_ROOT + - _APP_CONSOLE_WHITELIST_EMAILS + - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_HOSTNAMES + - _APP_SYSTEM_EMAIL_NAME + - _APP_SYSTEM_EMAIL_ADDRESS + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_SYSTEM_RESPONSE_FORMAT + - _APP_OPTIONS_ABUSE + - _APP_OPTIONS_ROUTER_PROTECTION + - _APP_OPTIONS_FORCE_HTTPS + - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS + - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_DOMAIN_FUNCTIONS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_SMTP_HOST + - _APP_SMTP_PORT + - _APP_SMTP_SECURE + - _APP_SMTP_USERNAME + - _APP_SMTP_PASSWORD + - _APP_USAGE_STATS + - _APP_STORAGE_LIMIT + - _APP_STORAGE_PREVIEW_LIMIT + - _APP_STORAGE_ANTIVIRUS + - _APP_STORAGE_ANTIVIRUS_HOST + - _APP_STORAGE_ANTIVIRUS_PORT + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET + - _APP_FUNCTIONS_SIZE_LIMIT + - _APP_FUNCTIONS_TIMEOUT + - _APP_FUNCTIONS_BUILD_TIMEOUT + - _APP_FUNCTIONS_CPUS + - _APP_FUNCTIONS_MEMORY + - _APP_FUNCTIONS_RUNTIMES + - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_HOST + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_MAINTENANCE_INTERVAL + - _APP_MAINTENANCE_DELAY + - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE + - _APP_MAINTENANCE_RETENTION_ABUSE + - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY + - _APP_MAINTENANCE_RETENTION_SCHEDULES + - _APP_SMS_PROVIDER + - _APP_SMS_FROM + - _APP_GRAPHQL_MAX_BATCH_SIZE + - _APP_GRAPHQL_MAX_COMPLEXITY + - _APP_GRAPHQL_MAX_DEPTH + - _APP_VCS_GITHUB_APP_NAME + - _APP_VCS_GITHUB_PRIVATE_KEY + - _APP_VCS_GITHUB_APP_ID + - _APP_VCS_GITHUB_WEBHOOK_SECRET + - _APP_VCS_GITHUB_CLIENT_SECRET + - _APP_VCS_GITHUB_CLIENT_ID + - _APP_ASSISTANT_OPENAI_API_KEY - tests: - build: - context: . - target: tests + appwrite-realtime: + image: appwrite/appwrite:1.5.7 + entrypoint: realtime + container_name: appwrite-realtime + restart: unless-stopped networks: - - tests - volumes: - - ./tests:/app/tests - - ./src:/app/src - - ./phpunit.xml:/app/phpunit.xml - working_dir: /app + - appwrite depends_on: - - supabase-db - - nhost-db - - nhost-storage + - mariadb + - redis environment: - - NHOST_DB_URL=postgres://postgres:postgres@nhost-db:5432/postgres - - SUPABASE_DB_URL=postgres://postgres:postgres@supabase-db:5432/postgres + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPTIONS_ABUSE + - _APP_OPTIONS_ROUTER_PROTECTION + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-audits: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-audits + container_name: appwrite-worker-audits + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-webhooks: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-webhooks + container_name: appwrite-worker-webhooks + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-deletes: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-deletes + container_name: appwrite-worker-deletes + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_HOST + + appwrite-worker-databases: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-databases + container_name: appwrite-worker-databases + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-builds: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-builds + container_name: appwrite-worker-builds + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_HOST + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_VCS_GITHUB_APP_NAME + - _APP_VCS_GITHUB_PRIVATE_KEY + - _APP_VCS_GITHUB_APP_ID + - _APP_FUNCTIONS_TIMEOUT + - _APP_FUNCTIONS_BUILD_TIMEOUT + - _APP_FUNCTIONS_CPUS + - _APP_FUNCTIONS_MEMORY + - _APP_FUNCTIONS_SIZE_LIMIT + - _APP_OPTIONS_FORCE_HTTPS + - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS + - _APP_DOMAIN + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET + + appwrite-worker-certificates: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-certificates + container_name: appwrite-worker-certificates + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_DOMAIN_FUNCTIONS + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-functions: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-functions + container_name: appwrite-worker-functions + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + - openruntimes-executor + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_FUNCTIONS_TIMEOUT + - _APP_FUNCTIONS_BUILD_TIMEOUT + - _APP_FUNCTIONS_CPUS + - _APP_FUNCTIONS_MEMORY + - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_HOST + - _APP_USAGE_STATS + - _APP_DOCKER_HUB_USERNAME + - _APP_DOCKER_HUB_PASSWORD + - _APP_LOGGING_CONFIG + - _APP_LOGGING_PROVIDER + + appwrite-worker-mails: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-mails + container_name: appwrite-worker-mails + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_SYSTEM_EMAIL_NAME + - _APP_SYSTEM_EMAIL_ADDRESS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_SMTP_HOST + - _APP_SMTP_PORT + - _APP_SMTP_SECURE + - _APP_SMTP_USERNAME + - _APP_SMTP_PASSWORD + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-worker-messaging: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-messaging + container_name: appwrite-worker-messaging + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_SMS_FROM + - _APP_SMS_PROVIDER + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET + + appwrite-worker-migrations: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-migrations + container_name: appwrite-worker-migrations + restart: unless-stopped + networks: + - appwrite + depends_on: + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_MIGRATIONS_FIREBASE_CLIENT_ID + - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET + + appwrite-task-maintenance: + image: appwrite/appwrite:1.5.7 + entrypoint: maintenance + container_name: appwrite-task-maintenance + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_DOMAIN_FUNCTIONS + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_MAINTENANCE_INTERVAL + - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_MAINTENANCE_RETENTION_CACHE + - _APP_MAINTENANCE_RETENTION_ABUSE + - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY + - _APP_MAINTENANCE_RETENTION_SCHEDULES + + appwrite-worker-usage: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-usage + container_name: appwrite-worker-usage + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + + appwrite-worker-usage-dump: + image: appwrite/appwrite:1.5.7 + entrypoint: worker-usage-dump + container_name: appwrite-worker-usage-dump + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + + appwrite-task-scheduler-functions: + image: appwrite/appwrite:1.5.7 + entrypoint: schedule-functions + container_name: appwrite-task-scheduler-functions + restart: unless-stopped + networks: + - appwrite + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + + appwrite-task-scheduler-messages: + image: appwrite/appwrite:1.5.7 + entrypoint: schedule-messages + container_name: appwrite-task-scheduler-messages + restart: unless-stopped + networks: + - appwrite + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + + appwrite-assistant: + image: appwrite/assistant:0.4.0 + container_name: appwrite-assistant + + restart: unless-stopped + networks: + - appwrite + environment: + - _APP_ASSISTANT_OPENAI_API_KEY + + openruntimes-executor: + container_name: openruntimes-executor + hostname: exc1 + restart: unless-stopped + stop_signal: SIGINT + image: openruntimes/executor:0.5.5 + networks: + - appwrite + - runtimes + volumes: + - /var/run/docker.sock:/var/run/docker.sock + # Host mount nessessary to share files between executor and runtimes. + # It's not possible to share mount file between 2 containers without host mount (copying is too slow) + - /tmp:/tmp:rw + environment: + - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL + - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK + - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME + - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD + - OPR_EXECUTOR_ENV=$_APP_ENV + - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES + - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET + - OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER + - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG + - OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE + - OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY + - OPR_EXECUTOR_STORAGE_S3_SECRET=$_APP_STORAGE_S3_SECRET + - OPR_EXECUTOR_STORAGE_S3_REGION=$_APP_STORAGE_S3_REGION + - OPR_EXECUTOR_STORAGE_S3_BUCKET=$_APP_STORAGE_S3_BUCKET + - OPR_EXECUTOR_STORAGE_DO_SPACES_ACCESS_KEY=$_APP_STORAGE_DO_SPACES_ACCESS_KEY + - OPR_EXECUTOR_STORAGE_DO_SPACES_SECRET=$_APP_STORAGE_DO_SPACES_SECRET + - OPR_EXECUTOR_STORAGE_DO_SPACES_REGION=$_APP_STORAGE_DO_SPACES_REGION + - OPR_EXECUTOR_STORAGE_DO_SPACES_BUCKET=$_APP_STORAGE_DO_SPACES_BUCKET + - OPR_EXECUTOR_STORAGE_BACKBLAZE_ACCESS_KEY=$_APP_STORAGE_BACKBLAZE_ACCESS_KEY + - OPR_EXECUTOR_STORAGE_BACKBLAZE_SECRET=$_APP_STORAGE_BACKBLAZE_SECRET + - OPR_EXECUTOR_STORAGE_BACKBLAZE_REGION=$_APP_STORAGE_BACKBLAZE_REGION + - OPR_EXECUTOR_STORAGE_BACKBLAZE_BUCKET=$_APP_STORAGE_BACKBLAZE_BUCKET + - OPR_EXECUTOR_STORAGE_LINODE_ACCESS_KEY=$_APP_STORAGE_LINODE_ACCESS_KEY + - OPR_EXECUTOR_STORAGE_LINODE_SECRET=$_APP_STORAGE_LINODE_SECRET + - OPR_EXECUTOR_STORAGE_LINODE_REGION=$_APP_STORAGE_LINODE_REGION + - OPR_EXECUTOR_STORAGE_LINODE_BUCKET=$_APP_STORAGE_LINODE_BUCKET + - OPR_EXECUTOR_STORAGE_WASABI_ACCESS_KEY=$_APP_STORAGE_WASABI_ACCESS_KEY + - OPR_EXECUTOR_STORAGE_WASABI_SECRET=$_APP_STORAGE_WASABI_SECRET + - OPR_EXECUTOR_STORAGE_WASABI_REGION=$_APP_STORAGE_WASABI_REGION + - OPR_EXECUTOR_STORAGE_WASABI_BUCKET=$_APP_STORAGE_WASABI_BUCKET + + mariadb: + image: mariadb:10.11 + container_name: appwrite-mariadb + + restart: unless-stopped + networks: + - appwrite + environment: + - MYSQL_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} + - MYSQL_DATABASE=${_APP_DB_SCHEMA} + - MYSQL_USER=${_APP_DB_USER} + - MYSQL_PASSWORD=${_APP_DB_PASS} + - MARIADB_AUTO_UPGRADE=1 + command: 'mysqld --innodb-flush-method=fsync' + + redis: + image: redis:7.2.4-alpine + container_name: appwrite-redis + restart: unless-stopped + command: > + redis-server + --maxmemory 512mb + --maxmemory-policy allkeys-lru + --maxmemory-samples 5 + networks: + - appwrite networks: tests: + name: tests + appwrite: + name: appwrite + runtimes: + name: runtimes diff --git a/tests/Migration/E2E/Adapters/Mock.php b/tests/Migration/E2E/Adapters/MockDestination.php similarity index 93% rename from tests/Migration/E2E/Adapters/Mock.php rename to tests/Migration/E2E/Adapters/MockDestination.php index e2c318a0..dc77d751 100644 --- a/tests/Migration/E2E/Adapters/Mock.php +++ b/tests/Migration/E2E/Adapters/MockDestination.php @@ -5,13 +5,13 @@ use Utopia\Migration\Destination; use Utopia\Migration\Resource; -class Mock extends Destination +class MockDestination extends Destination { public array $data = []; public static function getName(): string { - return 'Mock'; + return 'Mock Destination'; } public static function getSupportedResources(): array @@ -37,7 +37,7 @@ public static function getSupportedResources(): array public function import(array $resources, callable $callback): void { foreach ($resources as $resource) { - /** @var resource $resource */ + /** @var Resource $resource */ switch ($resource->getName()) { case 'Deployment': /** @var Deployment $resource */ diff --git a/tests/Migration/E2E/Sources/AppwriteTest.php b/tests/Migration/E2E/Sources/AppwriteTest.php new file mode 100644 index 00000000..d31b5102 --- /dev/null +++ b/tests/Migration/E2E/Sources/AppwriteTest.php @@ -0,0 +1,143 @@ + 0) { + try { + $this->call('GET', 'http://localhost:8000/v1'); + break; + } catch (\Exception $e) { + } + + sleep(5); + $tries--; + } + + // Bootstrap Appwrite + Console::execute( + "appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1", + '', + '' + ); + + // Run Faker + Console::execute( + "appwrite-toolkit --endpoint http://appwrite/v1 --auto faker", + '', + '' + ); + + // Parse Faker JSON + $projects = json_decode(file_get_contents('projects.json'), true); + $project = $projects[0]; + + // Create Appwrite Source + $this->source = new Appwrite($project['$id'], 'http://localhost:8000', $project['key']); + + $this->destination = new MockDestination(); + $this->transfer = new Transfer($this->source, $this->destination); + } + + public function testSourceReport() + { + // Test report all + $report = $this->source->report(); + + $this->assertNotEmpty($report); + + return [ + 'report' => $report, + ]; + } + + /** + * @depends testSourceReport + */ + public function testRunTransfer($state) + { + $this->transfer->run( + $this->source->getSupportedResources(), + function () { + } + ); + + $this->assertEquals(0, count($this->transfer->getReport('error'))); + + return array_merge($state, [ + 'transfer' => $this->transfer, + 'source' => $this->source, + ]); + } + + /** + * @depends testRunTransfer + */ + public function testValidateTransfer($state) + { + $statusCounters = $state['transfer']->getStatusCounters(); + $this->assertNotEmpty($statusCounters); + + foreach ($statusCounters as $resource => $counters) { + $this->assertNotEmpty($counters); + + if ($counters[Resource::STATUS_ERROR] > 0) { + $this->fail('Resource ' . $resource . ' has ' . $counters[Resource::STATUS_ERROR] . ' errors'); + + return; + } + } + + return $state; + } + + /** + * @depends testValidateTransfer + */ + public function testValidateUsersTransfer($state): void + { + // Process all users from Appwrite source and check if our copy is 1:1 + } + + /** + * @depends testValidateTransfer + */ + public function testValidateDatabaseTransfer($state): void + { + // Check each resource and make sure it's 1:1 + + // Databases + + // Collections + + // Attributes + + // Indexes + + // Documents + } + + /** + * @depends testValidateTransfer + */ + public function testValidateStorageTransfer($state): void + { + // Check each resource and make sure it's 1:1 + + // Validate Buckets + + // Validate Files + } +} diff --git a/tests/Migration/E2E/Sources/NHostTest.php b/tests/Migration/E2E/Sources/NHostTest.php index 4412b3fa..b826f403 100644 --- a/tests/Migration/E2E/Sources/NHostTest.php +++ b/tests/Migration/E2E/Sources/NHostTest.php @@ -2,20 +2,14 @@ namespace Utopia\Tests\E2E\Sources; -use Utopia\Migration\Destination; use Utopia\Migration\Resource; -use Utopia\Migration\Source; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Transfer; -use Utopia\Tests\E2E\Adapters\Mock; +use Utopia\Tests\E2E\Adapters\MockDestination; class NHostTest extends Base { - protected ?Source $source = null; - - protected ?Transfer $transfer = null; - - protected ?Destination $destination = null; + protected ?NHost $source = null; protected function setUp(): void { @@ -73,7 +67,7 @@ protected function setUp(): void $this->source->pdo = new \PDO('pgsql:host=nhost-db'.';port=5432;dbname=postgres', 'postgres', 'postgres'); $this->source->storageURL = 'http://nhost-storage'; - $this->destination = new Mock(); + $this->destination = new MockDestination(); $this->transfer = new Transfer($this->source, $this->destination); } diff --git a/tests/Migration/E2E/Sources/SupabaseTest.php b/tests/Migration/E2E/Sources/SupabaseTest.php index 66b80255..12b8d428 100644 --- a/tests/Migration/E2E/Sources/SupabaseTest.php +++ b/tests/Migration/E2E/Sources/SupabaseTest.php @@ -2,20 +2,14 @@ namespace Utopia\Tests\E2E\Sources; -use Utopia\Migration\Destination; use Utopia\Migration\Resource; -use Utopia\Migration\Source; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; -use Utopia\Tests\E2E\Adapters\Mock; +use Utopia\Tests\E2E\Adapters\MockDestination; class SupabaseTest extends Base { - protected ?Source $source = null; - - protected ?Transfer $transfer = null; - - protected ?Destination $destination = null; + protected ?Supabase $source = null; protected function setUp(): void { @@ -53,7 +47,7 @@ protected function setUp(): void 'postgres' ); - $this->destination = new Mock(); + $this->destination = new MockDestination(); $this->transfer = new Transfer($this->source, $this->destination); } From d3683e45085df36599b8599dab947643daff8e34 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 16:54:10 +0900 Subject: [PATCH 02/10] Run Linter and add envvars for Appwrite --- .env | 109 ++++++++++++++++++ .gitignore | 1 - bin/.env | 27 +++++ .../E2E/Adapters/MockDestination.php | 2 +- tests/Migration/E2E/Sources/AppwriteTest.php | 6 +- 5 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 .env create mode 100644 bin/.env diff --git a/.env b/.env new file mode 100644 index 00000000..0e267ec3 --- /dev/null +++ b/.env @@ -0,0 +1,109 @@ + +# Appwrite +_APP_ENV=development +_APP_EDITION=self-hosted +_APP_LOCALE=en +_APP_WORKER_PER_CORE=6 +_APP_CONSOLE_WHITELIST_ROOT=disabled +_APP_CONSOLE_WHITELIST_EMAILS= +_APP_CONSOLE_WHITELIST_IPS= +_APP_CONSOLE_COUNTRIES_DENYLIST=AQ +_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io +_APP_SYSTEM_EMAIL_NAME=Appwrite +_APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io +_APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io +_APP_SYSTEM_RESPONSE_FORMAT= +_APP_OPTIONS_ABUSE=disabled +_APP_OPTIONS_ROUTER_PROTECTION=disabled +_APP_OPTIONS_FORCE_HTTPS=disabled +_APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled +_APP_OPENSSL_KEY_V1=your-secret-key +_APP_DOMAIN=localhost +_APP_DOMAIN_FUNCTIONS=functions.localhost +_APP_DOMAIN_TARGET=localhost +_APP_REDIS_HOST=redis +_APP_REDIS_PORT=6379 +_APP_REDIS_PASS= +_APP_REDIS_USER= +_APP_DB_HOST=mariadb +_APP_DB_PORT=3306 +_APP_DB_SCHEMA=appwrite +_APP_DB_USER=user +_APP_DB_PASS=password +_APP_DB_ROOT_PASS=rootsecretpassword +_APP_STORAGE_DEVICE=Local +_APP_STORAGE_S3_ACCESS_KEY= +_APP_STORAGE_S3_SECRET= +_APP_STORAGE_S3_REGION=us-east-1 +_APP_STORAGE_S3_BUCKET= +_APP_STORAGE_DO_SPACES_ACCESS_KEY= +_APP_STORAGE_DO_SPACES_SECRET= +_APP_STORAGE_DO_SPACES_REGION=us-east-1 +_APP_STORAGE_DO_SPACES_BUCKET= +_APP_STORAGE_BACKBLAZE_ACCESS_KEY= +_APP_STORAGE_BACKBLAZE_SECRET= +_APP_STORAGE_BACKBLAZE_REGION=us-west-004 +_APP_STORAGE_BACKBLAZE_BUCKET= +_APP_STORAGE_LINODE_ACCESS_KEY= +_APP_STORAGE_LINODE_SECRET= +_APP_STORAGE_LINODE_REGION=eu-central-1 +_APP_STORAGE_LINODE_BUCKET= +_APP_STORAGE_WASABI_ACCESS_KEY= +_APP_STORAGE_WASABI_SECRET= +_APP_STORAGE_WASABI_REGION=eu-central-1 +_APP_STORAGE_WASABI_BUCKET= +_APP_STORAGE_ANTIVIRUS=disabled +_APP_STORAGE_ANTIVIRUS_HOST=clamav +_APP_STORAGE_ANTIVIRUS_PORT=3310 +_APP_SMTP_HOST=maildev +_APP_SMTP_PORT=1025 +_APP_SMTP_SECURE= +_APP_SMTP_USERNAME= +_APP_SMTP_PASSWORD= +_APP_SMS_PROVIDER=sms://username:password@mock +_APP_SMS_FROM=+123456789 +_APP_SMS_PROJECTS_DENY_LIST= +_APP_STORAGE_LIMIT=30000000 +_APP_STORAGE_PREVIEW_LIMIT=20000000 +_APP_FUNCTIONS_SIZE_LIMIT=30000000 +_APP_FUNCTIONS_TIMEOUT=900 +_APP_FUNCTIONS_BUILD_TIMEOUT=900 +_APP_FUNCTIONS_CPUS=1 +_APP_FUNCTIONS_MEMORY=1024 +_APP_FUNCTIONS_INACTIVE_THRESHOLD=600 +_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600 +_APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes +_APP_EXECUTOR_SECRET=your-secret-key +_APP_EXECUTOR_HOST=http://proxy/v1 +_APP_FUNCTIONS_RUNTIMES=php-8.0,node-18.0,python-3.9,ruby-3.1 +_APP_MAINTENANCE_INTERVAL=86400 +_APP_MAINTENANCE_DELAY= +_APP_MAINTENANCE_RETENTION_CACHE=2592000 +_APP_MAINTENANCE_RETENTION_EXECUTION=1209600 +_APP_MAINTENANCE_RETENTION_ABUSE=86400 +_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_USAGE_AGGREGATION_INTERVAL=30 +_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 +_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_USAGE_STATS=enabled +_APP_LOGGING_PROVIDER= +_APP_LOGGING_CONFIG= +_APP_GRAPHQL_MAX_BATCH_SIZE=10 +_APP_GRAPHQL_MAX_COMPLEXITY=250 +_APP_GRAPHQL_MAX_DEPTH=4 +_APP_DOCKER_HUB_USERNAME= +_APP_DOCKER_HUB_PASSWORD= +_APP_VCS_GITHUB_APP_NAME= +_APP_VCS_GITHUB_PRIVATE_KEY=disabled +_APP_VCS_GITHUB_APP_ID= +_APP_VCS_GITHUB_CLIENT_ID= +_APP_VCS_GITHUB_CLIENT_SECRET= +_APP_VCS_GITHUB_WEBHOOK_SECRET= +_APP_MIGRATIONS_FIREBASE_CLIENT_ID= +_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= +_APP_ASSISTANT_OPENAI_API_KEY= +_APP_MESSAGE_SMS_TEST_DSN= +_APP_MESSAGE_EMAIL_TEST_DSN= +_APP_MESSAGE_PUSH_TEST_DSN= +_APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 +_APP_PROJECT_REGIONS=default \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9c0bf128..c2ae0821 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ composer.lock /vendor/ /.idea/ *.cache -.env test-service-account.json .DS_Store localBackup/ diff --git a/bin/.env b/bin/.env new file mode 100644 index 00000000..7ea0909c --- /dev/null +++ b/bin/.env @@ -0,0 +1,27 @@ +SOURCE=supabase + +SOURCE_APPWRITE_TEST_PROJECT=6646cbd3874a550d78e1 +SOURCE_APPWRITE_TEST_ENDPOINT=http://localhost/v1 +SOURCE_APPWRITE_TEST_KEY=d485cca1318af57b68213bcecc9defbeeccb1f3cf94774a4f097b61415f38b2f78f13faf2a3ec91930105df5b8266854f6f8219a8d693040a83ea600fd4458875f741fc7e318b3eca685c3c25a8523d765bcda9ec7f7e8129448cc5986b2fc2c7797300ae27ed68e764dbda508e82a66514fcf9da3187d0eb5f4453e031a1f7d + +FIREBASE_TEST_PROJECT=testProject +FIREBASE_TEST_ACCOUNT='{type: "service_account", ...}' + +SUPABASE_TEST_ENDPOINT=https://iwnzcwbimmpurmkqcueg.supabase.co +SUPABASE_TEST_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9 +SUPABASE_TEST_HOST=aws-0-eu-west-2.pooler.supabase.com +SUPABASE_TEST_DATABASE=postgres +SUPABASE_TEST_USERNAME=postgres.iwnzcwbimmpurmkqcueg +SUPABASE_TEST_PASSWORD=ueTlfhe4bcNfiqHV + +NHOST_TEST_SUBDOMAIN=xxxxxxxxxxx +NHOST_TEST_REGION=eu-central-1 +NHOST_TEST_SECRET=xxxxxxxxxxxxxxxxx +NHOST_TEST_DATABASE=xxxxxxxxxxxxxxx +NHOST_TEST_USERNAME=postgres +NHOST_TEST_PASSWORD=xxxxxxxxxxxxxxx + + +DESTINATION_APPWRITE_TEST_PROJECT=6646dd50000c4b4f2b04 +DESTINATION_APPWRITE_TEST_ENDPOINT=http://localhost/v1 +DESTINATION_APPWRITE_TEST_KEY=c8c3ee7f9491b8320cd57aaad08531a5bbd77651da95b166ee5c26b7f9081dc124952b37a75d1cc4302dd9439d884d7265ceb8c88563cbb3d5bc2bad19381d272233e3836cc7035a166fc75e405dc5a674eeee0c11e646e35848326870974f4ef29412ad97bd81e095fb9c2739fed079f7704a83d8924d199cd65d02534cdda9 diff --git a/tests/Migration/E2E/Adapters/MockDestination.php b/tests/Migration/E2E/Adapters/MockDestination.php index dc77d751..3232d621 100644 --- a/tests/Migration/E2E/Adapters/MockDestination.php +++ b/tests/Migration/E2E/Adapters/MockDestination.php @@ -37,7 +37,7 @@ public static function getSupportedResources(): array public function import(array $resources, callable $callback): void { foreach ($resources as $resource) { - /** @var Resource $resource */ + /** @var resource $resource */ switch ($resource->getName()) { case 'Deployment': /** @var Deployment $resource */ diff --git a/tests/Migration/E2E/Sources/AppwriteTest.php b/tests/Migration/E2E/Sources/AppwriteTest.php index d31b5102..976a3156 100644 --- a/tests/Migration/E2E/Sources/AppwriteTest.php +++ b/tests/Migration/E2E/Sources/AppwriteTest.php @@ -28,14 +28,14 @@ protected function setUp(): void // Bootstrap Appwrite Console::execute( - "appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1", + 'appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1', '', '' ); // Run Faker Console::execute( - "appwrite-toolkit --endpoint http://appwrite/v1 --auto faker", + 'appwrite-toolkit --endpoint http://appwrite/v1 --auto faker', '', '' ); @@ -94,7 +94,7 @@ public function testValidateTransfer($state) $this->assertNotEmpty($counters); if ($counters[Resource::STATUS_ERROR] > 0) { - $this->fail('Resource ' . $resource . ' has ' . $counters[Resource::STATUS_ERROR] . ' errors'); + $this->fail('Resource '.$resource.' has '.$counters[Resource::STATUS_ERROR].' errors'); return; } From 3f6664a852ecbd6357beb00e5482416e84006dcd Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 16:56:15 +0900 Subject: [PATCH 03/10] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a23bb9c..0c053481 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ FROM supabase/postgres:15.1.0.96 as supabase-db -COPY tests/Migration/Resources/supabase/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql -COPY tests/Migration/Resources/supabase/2_main.sql /docker-entrypoint-initdb.d/2_main.sql +COPY tests/Migration/resources/supabase/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql +COPY tests/Migration/resources/supabase/2_main.sql /docker-entrypoint-initdb.d/2_main.sql RUN rm -rf /docker-entrypoint-initdb.d/migrate.sh FROM postgres:alpine3.18 as nhost-db -COPY tests/Migration/Resources/nhost/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql -COPY tests/Migration/Resources/nhost/2_main.sql /docker-entrypoint-initdb.d/2_main.sql +COPY tests/Migration/resources/nhost/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql +COPY tests/Migration/resources/nhost/2_main.sql /docker-entrypoint-initdb.d/2_main.sql FROM composer:2.0 as composer From e44810296068688fc3efa5810707cd58bec766d1 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 16:58:26 +0900 Subject: [PATCH 04/10] Add Image Caching --- .github/workflows/tests.yml | 38 ++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 19069cce..e31aaba1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,35 @@ on: push: { branches: [main] } jobs: + setup: + name: Setup & Build Migrations Image + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Migrations + uses: docker/build-push-action@v3 + with: + context: . + push: false + tags: ${{ env.IMAGE }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar + + - name: Cache Docker Image + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + source_e2e_tests: name: Run Source E2E Tests runs-on: ubuntu-latest @@ -17,9 +46,16 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Load Cache + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + - name: Bring up services run: docker-compose up -d --build - name: Run ${{matrix.adapter}} Tests run: | - docker compose exec tests php ./vendor/bin/phpunit /usr/src/code/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php \ No newline at end of file + docker compose exec tests php ./vendor/bin/phpunit /usr/src/code/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php From 07bd0a979b70e2558aed8d8fdd2782ab0abceeb7 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 17:03:37 +0900 Subject: [PATCH 05/10] Update tests.yml --- .github/workflows/tests.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e31aaba1..d8b5112c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,13 @@ name: Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: migrations-dev + CACHE_KEY: migrations-dev-${{ github.event.pull_request.head.sha }} + on: pull_request: push: { branches: [main] } @@ -36,6 +45,7 @@ jobs: source_e2e_tests: name: Run Source E2E Tests runs-on: ubuntu-latest + needs: setup env: COMPOSE_FILE: docker-compose.yml strategy: @@ -54,8 +64,8 @@ jobs: fail-on-cache-miss: true - name: Bring up services - run: docker-compose up -d --build + run: docker-compose up -d - name: Run ${{matrix.adapter}} Tests run: | - docker compose exec tests php ./vendor/bin/phpunit /usr/src/code/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php + docker compose exec tests php ./vendor/bin/phpunit /app/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php From a8a195a617d98750574e359191f11a9f9b5610fd Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 17:10:21 +0900 Subject: [PATCH 06/10] Continue trying to get the cache to work --- .github/workflows/tests.yml | 22 +++++++++++++++++++--- docker-compose.yml | 1 + 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8b5112c..d2e685c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,7 @@ concurrency: env: IMAGE: migrations-dev - CACHE_KEY: migrations-dev-${{ github.event.pull_request.head.sha }} + CACHE_KEY: migrations-dev-${{ github.sha }} on: pull_request: @@ -25,6 +25,22 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Debug Cache + run: | + if [ -f /tmp/${{ env.IMAGE }}.tar ]; then + echo "Cache hit" + else + echo "Cache miss" + fi + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ env.CACHE_KEY }}-docker + restore-keys: | + ${{ env.CACHE_KEY }}-docker + - name: Build Migrations uses: docker/build-push-action@v3 with: @@ -32,8 +48,8 @@ jobs: push: false tags: ${{ env.IMAGE }} load: true - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar - name: Cache Docker Image diff --git a/docker-compose.yml b/docker-compose.yml index 441b5a44..2f383c56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: ## Tests tests: + image: migrations-dev build: context: . target: tests From eba675fa041a591734d5c0180bad90e83488cf71 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Sat, 1 Jun 2024 17:21:35 +0900 Subject: [PATCH 07/10] Continue work on tests --- .github/workflows/tests.yml | 24 +++----------------- tests/Migration/E2E/Sources/AppwriteTest.php | 17 ++++++++++++-- tests/Migration/E2E/Sources/NHostTest.php | 3 +-- tests/Migration/E2E/Sources/SupabaseTest.php | 2 -- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d2e685c2..2f6f9216 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,32 +25,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Debug Cache - run: | - if [ -f /tmp/${{ env.IMAGE }}.tar ]; then - echo "Cache hit" - else - echo "Cache miss" - fi - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ env.CACHE_KEY }}-docker - restore-keys: | - ${{ env.CACHE_KEY }}-docker - - name: Build Migrations - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . push: false tags: ${{ env.IMAGE }} - load: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar + cache-from: type=gha + cache-to: type=gha,mode=max - name: Cache Docker Image uses: actions/cache@v3 diff --git a/tests/Migration/E2E/Sources/AppwriteTest.php b/tests/Migration/E2E/Sources/AppwriteTest.php index 976a3156..3e45298a 100644 --- a/tests/Migration/E2E/Sources/AppwriteTest.php +++ b/tests/Migration/E2E/Sources/AppwriteTest.php @@ -27,19 +27,32 @@ protected function setUp(): void } // Bootstrap Appwrite + $stdout = ''; + $stderr = ''; Console::execute( 'appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1', '', - '' + $stdout, + $stderr ); + Console::info($stdout); + Console::error($stderr); + + $stdout = ''; + $stderr = ''; + // Run Faker Console::execute( 'appwrite-toolkit --endpoint http://appwrite/v1 --auto faker', '', - '' + $stdout, + $stderr ); + Console::info($stdout); + Console::error($stderr); + // Parse Faker JSON $projects = json_decode(file_get_contents('projects.json'), true); $project = $projects[0]; diff --git a/tests/Migration/E2E/Sources/NHostTest.php b/tests/Migration/E2E/Sources/NHostTest.php index b826f403..aa1876fe 100644 --- a/tests/Migration/E2E/Sources/NHostTest.php +++ b/tests/Migration/E2E/Sources/NHostTest.php @@ -9,8 +9,6 @@ class NHostTest extends Base { - protected ?NHost $source = null; - protected function setUp(): void { // Check DB is online and ready @@ -64,6 +62,7 @@ protected function setUp(): void 'postgres', 'password' ); + $this->source->pdo = new \PDO('pgsql:host=nhost-db'.';port=5432;dbname=postgres', 'postgres', 'postgres'); $this->source->storageURL = 'http://nhost-storage'; diff --git a/tests/Migration/E2E/Sources/SupabaseTest.php b/tests/Migration/E2E/Sources/SupabaseTest.php index 12b8d428..ba23a161 100644 --- a/tests/Migration/E2E/Sources/SupabaseTest.php +++ b/tests/Migration/E2E/Sources/SupabaseTest.php @@ -9,8 +9,6 @@ class SupabaseTest extends Base { - protected ?Supabase $source = null; - protected function setUp(): void { // Check DB is online and ready From 75ec3cc9f66a7bb7dae69116ebbfdcf7ca04cbb9 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 3 Jun 2024 12:42:33 +0900 Subject: [PATCH 08/10] Rename tests to unit --- .github/workflows/tests.yml | 6 +- Dockerfile | 12 + dev/xdebug.ini | 6 + docker-compose.yml | 4 +- phpunit.xml | 4 +- src/Migration/Transfer.php | 16 ++ tests/Migration/E2E/Sources/AppwriteTest.php | 156 ------------ .../Adapters/MockDestination.php | 22 +- tests/Migration/Unit/Sources/AppwriteTest.php | 237 ++++++++++++++++++ .../Migration/{E2E => Unit}/Sources/Base.php | 9 +- .../{E2E => Unit}/Sources/NHostTest.php | 4 +- .../{E2E => Unit}/Sources/SupabaseTest.php | 4 +- 12 files changed, 306 insertions(+), 174 deletions(-) create mode 100644 dev/xdebug.ini delete mode 100644 tests/Migration/E2E/Sources/AppwriteTest.php rename tests/Migration/{E2E => Unit}/Adapters/MockDestination.php (72%) create mode 100644 tests/Migration/Unit/Sources/AppwriteTest.php rename tests/Migration/{E2E => Unit}/Sources/Base.php (95%) rename tests/Migration/{E2E => Unit}/Sources/NHostTest.php (98%) rename tests/Migration/{E2E => Unit}/Sources/SupabaseTest.php (98%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2f6f9216..2e0b5cab 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,8 +40,8 @@ jobs: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar - source_e2e_tests: - name: Run Source E2E Tests + source_unit_tests: + name: Run Source Unit Tests runs-on: ubuntu-latest needs: setup env: @@ -66,4 +66,4 @@ jobs: - name: Run ${{matrix.adapter}} Tests run: | - docker compose exec tests php ./vendor/bin/phpunit /app/tests/Migration/E2E/Sources/${{matrix.adapter}}Test.php + docker compose exec tests php ./vendor/bin/phpunit /app/tests/Migration/Unit/Sources/${{matrix.adapter}}Test.php diff --git a/Dockerfile b/Dockerfile index 0c053481..468bf2fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,18 @@ RUN set -ex \ && docker-php-ext-install pdo pdo_pgsql \ && apt-get remove -y libpq-dev +## Install XDebug, Remove before commit. +RUN \ + git clone --depth 1 --branch 3.3.1 https://github.com/xdebug/xdebug && \ + cd xdebug && \ + phpize && \ + ./configure && \ + make && make install + +# Enable Extensions +COPY dev/xdebug.ini /usr/src/code/dev/xdebug.ini +RUN cp /usr/src/code/dev/xdebug.ini /usr/local/etc/php/conf.d/xdebug.ini + COPY ./src /app/src COPY ./tests /app/src/tests COPY --from=composer /usr/local/src/vendor /app/vendor diff --git a/dev/xdebug.ini b/dev/xdebug.ini new file mode 100644 index 00000000..e29c8bd4 --- /dev/null +++ b/dev/xdebug.ini @@ -0,0 +1,6 @@ +zend_extension=xdebug + +[xdebug] +xdebug.mode=develop,debug +xdebug.client_host=host.docker.internal +xdebug.start_with_request=yes \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2f383c56..8118dfd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: ## Tests tests: - image: migrations-dev + image: migration-tests build: context: . target: tests @@ -22,6 +22,7 @@ services: ## Supabase supabase-db: + image: migration-supabase build: context: . target: supabase-db @@ -44,6 +45,7 @@ services: ## NHost nhost-db: + image: migration-nhost build: context: . target: nhost-db diff --git a/phpunit.xml b/phpunit.xml index 9a350007..9dbb40b5 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -9,8 +9,8 @@ stopOnFailure="false" > - - ./tests/Migration/E2E + + ./tests/Migration/Unit diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 6049ec9a..7a10114a 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -203,4 +203,20 @@ public function getReport(string $statusLevel = ''): array return $report; } + + /** + * Get Source + */ + public function getSource(): Source + { + return $this->source; + } + + /** + * Get Destination + */ + public function getDestination(): Destination + { + return $this->destination; + } } diff --git a/tests/Migration/E2E/Sources/AppwriteTest.php b/tests/Migration/E2E/Sources/AppwriteTest.php deleted file mode 100644 index 3e45298a..00000000 --- a/tests/Migration/E2E/Sources/AppwriteTest.php +++ /dev/null @@ -1,156 +0,0 @@ - 0) { - try { - $this->call('GET', 'http://localhost:8000/v1'); - break; - } catch (\Exception $e) { - } - - sleep(5); - $tries--; - } - - // Bootstrap Appwrite - $stdout = ''; - $stderr = ''; - Console::execute( - 'appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1', - '', - $stdout, - $stderr - ); - - Console::info($stdout); - Console::error($stderr); - - $stdout = ''; - $stderr = ''; - - // Run Faker - Console::execute( - 'appwrite-toolkit --endpoint http://appwrite/v1 --auto faker', - '', - $stdout, - $stderr - ); - - Console::info($stdout); - Console::error($stderr); - - // Parse Faker JSON - $projects = json_decode(file_get_contents('projects.json'), true); - $project = $projects[0]; - - // Create Appwrite Source - $this->source = new Appwrite($project['$id'], 'http://localhost:8000', $project['key']); - - $this->destination = new MockDestination(); - $this->transfer = new Transfer($this->source, $this->destination); - } - - public function testSourceReport() - { - // Test report all - $report = $this->source->report(); - - $this->assertNotEmpty($report); - - return [ - 'report' => $report, - ]; - } - - /** - * @depends testSourceReport - */ - public function testRunTransfer($state) - { - $this->transfer->run( - $this->source->getSupportedResources(), - function () { - } - ); - - $this->assertEquals(0, count($this->transfer->getReport('error'))); - - return array_merge($state, [ - 'transfer' => $this->transfer, - 'source' => $this->source, - ]); - } - - /** - * @depends testRunTransfer - */ - public function testValidateTransfer($state) - { - $statusCounters = $state['transfer']->getStatusCounters(); - $this->assertNotEmpty($statusCounters); - - foreach ($statusCounters as $resource => $counters) { - $this->assertNotEmpty($counters); - - if ($counters[Resource::STATUS_ERROR] > 0) { - $this->fail('Resource '.$resource.' has '.$counters[Resource::STATUS_ERROR].' errors'); - - return; - } - } - - return $state; - } - - /** - * @depends testValidateTransfer - */ - public function testValidateUsersTransfer($state): void - { - // Process all users from Appwrite source and check if our copy is 1:1 - } - - /** - * @depends testValidateTransfer - */ - public function testValidateDatabaseTransfer($state): void - { - // Check each resource and make sure it's 1:1 - - // Databases - - // Collections - - // Attributes - - // Indexes - - // Documents - } - - /** - * @depends testValidateTransfer - */ - public function testValidateStorageTransfer($state): void - { - // Check each resource and make sure it's 1:1 - - // Validate Buckets - - // Validate Files - } -} diff --git a/tests/Migration/E2E/Adapters/MockDestination.php b/tests/Migration/Unit/Adapters/MockDestination.php similarity index 72% rename from tests/Migration/E2E/Adapters/MockDestination.php rename to tests/Migration/Unit/Adapters/MockDestination.php index 3232d621..0c391c98 100644 --- a/tests/Migration/E2E/Adapters/MockDestination.php +++ b/tests/Migration/Unit/Adapters/MockDestination.php @@ -1,6 +1,6 @@ getName()) { case 'Deployment': /** @var Deployment $resource */ if ($resource->getStart() === 0) { - $this->data[$resource->getGroup()][$resource->getName()][$resource->getInternalId()] = $resource->asArray(); + $this->data[$resource->getName()][$resource->getInternalId()] = $resource->asArray(); } // file_put_contents($this->path . 'deployments/' . $resource->getId() . '.tar.gz', $resource->getData(), FILE_APPEND); @@ -50,6 +50,9 @@ public function import(array $resources, callable $callback): void case Resource::TYPE_FILE: /** @var File $resource */ break; + default: + $this->data[$resource->getName()][$resource->getId()] = $resource->asArray(); + break; } $resource->setStatus(Resource::STATUS_SUCCESS); @@ -63,4 +66,17 @@ public function report(array $groups = []): array { return []; } + + public function get(string $resource, string $id): ?array + { + if (!key_exists($resource, $this->data)) { + return null; + } + + if (!key_exists($id, $this->data[$resource])) { + return null; + } + + return $this->data[$resource][$id]; + } } diff --git a/tests/Migration/Unit/Sources/AppwriteTest.php b/tests/Migration/Unit/Sources/AppwriteTest.php new file mode 100644 index 00000000..49f2d7d9 --- /dev/null +++ b/tests/Migration/Unit/Sources/AppwriteTest.php @@ -0,0 +1,237 @@ + 0) { + if ($tries === 0) { + throw new \Exception('Appwrite was offline after 5 tries'); + } + + // Static doesn't have access to $this->call + $ch = curl_init('http://appwrite/v1/health'); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + + curl_exec($ch); + $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($status === 200) { + break; + } + + sleep(5); + $tries--; + } + + Console::info('Bootstrapping Appwrite...'); + + // Bootstrap Appwrite + $stdout = ''; + Console::execute( + 'appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1', + '', + $stdout, + ); + Console::info($stdout); + + Console::info('Running Faker...'); + + // Run Faker + $stdout = ''; + Console::execute( + 'appwrite-toolkit --endpoint http://appwrite/v1 --auto faker', + '', + $stdout + ); + Console::info($stdout); + + Console::info('Initial setup complete'); + } + + public function setup(): void + { + // Parse Faker JSON + $projects = json_decode(file_get_contents('projects.json'), true); + $project = $projects[0]; + + $this->source = new Appwrite( + $project['$id'], + 'http://appwrite/v1', + $project['apiKey'] + ); + + $this->client = new Client(); + $this->client + ->setEndpoint('http://appwrite/v1') + ->setProject($project['$id']) + ->setKey($project['apiKey']); + + $this->destination = new MockDestination(); + } + + public function testSourceReport() + { + // Test report all + $report = $this->source->report(); + + $this->assertNotEmpty($report); + + return [ + 'report' => $report, + ]; + } + + /** + * @depends testSourceReport + */ + public function testRunTransfer($state) + { + $transfer = new Transfer($this->source, $this->destination); + + $transfer->run( + $this->source->getSupportedResources(), + function () { + } + ); + + $this->assertEquals(0, count($transfer->getReport('error'))); + + return array_merge($state, [ + 'transfer' => $transfer, + 'source' => $this->source, + ]); + } + + /** + * @depends testRunTransfer + */ + public function testValidateTransfer($state) + { + $statusCounters = $state['transfer']->getStatusCounters(); + $this->assertNotEmpty($statusCounters); + + foreach ($statusCounters as $resource => $counters) { + $this->assertNotEmpty($counters); + + if ($counters[Resource::STATUS_ERROR] > 0) { + $this->fail('Resource ' . $resource . ' has ' . $counters[Resource::STATUS_ERROR] . ' errors'); + + return; + } + } + + return $state; + } + + /** + * @depends testValidateTransfer + */ + public function testValidateAuthTransfer($state): void + { + // Process all users from Appwrite source and check if our copy is 1:1 + $userClient = new Users($this->client); + + /** @var Transfer $transfer */ + $transfer = $state['transfer']; + + /** @var MockDestination $destination */ + $destination = $transfer->getDestination(); + + $last = ''; + while (true) { + $response = $userClient->list( + empty($last) ? [] : [Query::cursorAfter($last)] + ); + + foreach ($response['users'] as $user) { + // Check if exists + $destinationUser = $destination->get('user', $user['$id']); + + if (empty($destinationUser)) { + $this->fail('User ' . $user['$id'] . ' not found in destination'); + } + + // Compare data + $this->assertEquals($user['$id'], $destinationUser['id']); + $this->assertEquals($user['email'], $destinationUser['email']); + $this->assertEquals($user['name'], $destinationUser['username']); + $this->assertEquals($user['password'], $destinationUser['passwordHash']); + $this->assertEquals($user['phone'], $destinationUser['phone']); + $this->assertEquals($user['emailVerification'], $destinationUser['emailVerified']); + + $last = $user['$id']; + } + + if (empty($response['sum'])) { + break; + } + } + } + + /** + * @depends testValidateAuthTransfer + */ + public function testValidateDatabaseTransfer($state): void + { + // Check each resource and make sure it's 1:1 + + // Databases + + // Collections + + // Attributes + + // Indexes + + // Documents + } + + /** + * @depends testValidateDatabaseTransfer + */ + public function testValidateStorageTransfer($state): void + { + // Check each resource and make sure it's 1:1 + + // Validate Buckets + + // Validate Files + } + + /** + * Compare data between original and copy ignoring any fields that are not relevant + * + * @param array $original + * @param array $copy + * @param array $ignore + * + * @return bool + */ + private function compareData(array $original, array $copy, array $ignore) + { + } +} diff --git a/tests/Migration/E2E/Sources/Base.php b/tests/Migration/Unit/Sources/Base.php similarity index 95% rename from tests/Migration/E2E/Sources/Base.php rename to tests/Migration/Unit/Sources/Base.php index c5a7f231..47594fde 100644 --- a/tests/Migration/E2E/Sources/Base.php +++ b/tests/Migration/Unit/Sources/Base.php @@ -1,13 +1,12 @@ destination = new Mock(); + $this->destination = new MockDestination(); $this->transfer = new Transfer($this->source, $this->destination); } diff --git a/tests/Migration/E2E/Sources/NHostTest.php b/tests/Migration/Unit/Sources/NHostTest.php similarity index 98% rename from tests/Migration/E2E/Sources/NHostTest.php rename to tests/Migration/Unit/Sources/NHostTest.php index aa1876fe..37cc577b 100644 --- a/tests/Migration/E2E/Sources/NHostTest.php +++ b/tests/Migration/Unit/Sources/NHostTest.php @@ -1,11 +1,11 @@ Date: Mon, 3 Jun 2024 12:43:18 +0900 Subject: [PATCH 09/10] Run Linter --- tests/Migration/Unit/Adapters/MockDestination.php | 6 +++--- tests/Migration/Unit/Sources/AppwriteTest.php | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/Migration/Unit/Adapters/MockDestination.php b/tests/Migration/Unit/Adapters/MockDestination.php index 0c391c98..447732e6 100644 --- a/tests/Migration/Unit/Adapters/MockDestination.php +++ b/tests/Migration/Unit/Adapters/MockDestination.php @@ -37,7 +37,7 @@ public static function getSupportedResources(): array public function import(array $resources, callable $callback): void { foreach ($resources as $resource) { - /** @var Resource $resource */ + /** @var resource $resource */ switch ($resource->getName()) { case 'Deployment': /** @var Deployment $resource */ @@ -69,11 +69,11 @@ public function report(array $groups = []): array public function get(string $resource, string $id): ?array { - if (!key_exists($resource, $this->data)) { + if (! array_key_exists($resource, $this->data)) { return null; } - if (!key_exists($id, $this->data[$resource])) { + if (! array_key_exists($id, $this->data[$resource])) { return null; } diff --git a/tests/Migration/Unit/Sources/AppwriteTest.php b/tests/Migration/Unit/Sources/AppwriteTest.php index 49f2d7d9..2d93e119 100644 --- a/tests/Migration/Unit/Sources/AppwriteTest.php +++ b/tests/Migration/Unit/Sources/AppwriteTest.php @@ -20,6 +20,7 @@ public static function setUpBeforeClass(): void // If we've already bootstrapped Appwrite, skip if (file_exists('projects.json')) { Console::info('Appwrite already bootstrapped, skipping'); + return; } @@ -138,7 +139,7 @@ public function testValidateTransfer($state) $this->assertNotEmpty($counters); if ($counters[Resource::STATUS_ERROR] > 0) { - $this->fail('Resource ' . $resource . ' has ' . $counters[Resource::STATUS_ERROR] . ' errors'); + $this->fail('Resource '.$resource.' has '.$counters[Resource::STATUS_ERROR].' errors'); return; } @@ -154,7 +155,7 @@ public function testValidateAuthTransfer($state): void { // Process all users from Appwrite source and check if our copy is 1:1 $userClient = new Users($this->client); - + /** @var Transfer $transfer */ $transfer = $state['transfer']; @@ -172,7 +173,7 @@ public function testValidateAuthTransfer($state): void $destinationUser = $destination->get('user', $user['$id']); if (empty($destinationUser)) { - $this->fail('User ' . $user['$id'] . ' not found in destination'); + $this->fail('User '.$user['$id'].' not found in destination'); } // Compare data @@ -224,11 +225,8 @@ public function testValidateStorageTransfer($state): void /** * Compare data between original and copy ignoring any fields that are not relevant - * - * @param array $original - * @param array $copy - * @param array $ignore - * + * + * * @return bool */ private function compareData(array $original, array $copy, array $ignore) From 8d00c87375f81506aec05edc4e6c5693a6c84568 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 3 Jun 2024 13:43:18 +0900 Subject: [PATCH 10/10] Update AppwriteTest.php --- tests/Migration/Unit/Sources/AppwriteTest.php | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/Migration/Unit/Sources/AppwriteTest.php b/tests/Migration/Unit/Sources/AppwriteTest.php index 2d93e119..35eb8763 100644 --- a/tests/Migration/Unit/Sources/AppwriteTest.php +++ b/tests/Migration/Unit/Sources/AppwriteTest.php @@ -4,6 +4,7 @@ use Appwrite\Client; use Appwrite\Query; +use Appwrite\Services\Teams; use Appwrite\Services\Users; use Utopia\CLI\Console; use Utopia\Migration\Resource; @@ -17,7 +18,6 @@ class AppwriteTest extends Base public static function setUpBeforeClass(): void { - // If we've already bootstrapped Appwrite, skip if (file_exists('projects.json')) { Console::info('Appwrite already bootstrapped, skipping'); @@ -50,7 +50,6 @@ public static function setUpBeforeClass(): void Console::info('Bootstrapping Appwrite...'); - // Bootstrap Appwrite $stdout = ''; Console::execute( 'appwrite-toolkit --endpoint http://appwrite/v1 --auto bootstrap --amount 1', @@ -61,7 +60,6 @@ public static function setUpBeforeClass(): void Console::info('Running Faker...'); - // Run Faker $stdout = ''; Console::execute( 'appwrite-toolkit --endpoint http://appwrite/v1 --auto faker', @@ -139,7 +137,7 @@ public function testValidateTransfer($state) $this->assertNotEmpty($counters); if ($counters[Resource::STATUS_ERROR] > 0) { - $this->fail('Resource '.$resource.' has '.$counters[Resource::STATUS_ERROR].' errors'); + $this->fail('Resource ' . $resource . ' has ' . $counters[Resource::STATUS_ERROR] . ' errors'); return; } @@ -153,8 +151,8 @@ public function testValidateTransfer($state) */ public function testValidateAuthTransfer($state): void { - // Process all users from Appwrite source and check if our copy is 1:1 $userClient = new Users($this->client); + $teamClient = new Teams($this->client); /** @var Transfer $transfer */ $transfer = $state['transfer']; @@ -162,6 +160,7 @@ public function testValidateAuthTransfer($state): void /** @var MockDestination $destination */ $destination = $transfer->getDestination(); + // Check Users $last = ''; while (true) { $response = $userClient->list( @@ -173,7 +172,7 @@ public function testValidateAuthTransfer($state): void $destinationUser = $destination->get('user', $user['$id']); if (empty($destinationUser)) { - $this->fail('User '.$user['$id'].' not found in destination'); + $this->fail('User ' . $user['$id'] . ' not found in destination'); } // Compare data @@ -183,6 +182,7 @@ public function testValidateAuthTransfer($state): void $this->assertEquals($user['password'], $destinationUser['passwordHash']); $this->assertEquals($user['phone'], $destinationUser['phone']); $this->assertEquals($user['emailVerification'], $destinationUser['emailVerified']); + $this->assertEquals($user['phoneVerification'], $destinationUser['phoneVerified']); $last = $user['$id']; } @@ -191,6 +191,34 @@ public function testValidateAuthTransfer($state): void break; } } + + // Check Teams + $last = ''; + while (true) { + $response = $teamClient->list( + empty($last) ? [] : [Query::cursorAfter($last)] + ); + + foreach ($response['teams'] as $team) { + // Check if exists + $destinationTeam = $destination->get('team', $team['$id']); + + if (empty($destinationTeam)) { + $this->fail('Team ' . $team['$id'] . ' not found in destination'); + } + + // Compare data + $this->assertEquals($team['$id'], $destinationTeam['id']); + $this->assertEquals($team['name'], $destinationTeam['name']); + $this->assertEquals($team['prefs'], $destinationTeam['preferences']); + + $last = $team['$id']; + } + + if (empty($response['sum'])) { + break; + } + } } /**