From 340b95c653969b7aafe6c0eb215dab7e212c47d8 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 3 Mar 2026 18:10:54 +0100 Subject: [PATCH] ci: Add test with partitioned filecache on mariadb Signed-off-by: Carl Schwan --- .../workflows/phpunit-mariadb-partition.yml | 154 ++++++++++++++++++ .../Version13000Date20170718121200.php | 17 +- tests/preseed-config.php | 6 + 3 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/phpunit-mariadb-partition.yml diff --git a/.github/workflows/phpunit-mariadb-partition.yml b/.github/workflows/phpunit-mariadb-partition.yml new file mode 100644 index 0000000000000..ab544d274a41e --- /dev/null +++ b/.github/workflows/phpunit-mariadb-partition.yml @@ -0,0 +1,154 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization +# +# SPDX-FileCopyrightText: 2023-2024 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: MIT + +name: PHPUnit MariaDB + +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +permissions: + contents: read + +concurrency: + group: phpunit-mariadb-partition-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + permissions: + contents: read + pull-requests: read + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-mariadb: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + fail-fast: false + matrix: + php-versions: ['8.2'] + mariadb-versions: ['10.6'] + include: + - php-versions: '8.5' + mariadb-versions: '11.8' + + name: MariaDB Partitionning ${{ matrix.mariadb-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest # zizmor: ignore[unpinned-images] + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + mariadb: + image: mariadb:${{ matrix.mariadb-versions }} + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="${{ matrix.mariadb-versions <= 10.4 && 'mysqladmin' || 'mariadb-admin'}} ping" --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # v2.36.0 + timeout-minutes: 5 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Enable ONLY_FULL_GROUP_BY MariaDB option + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo 'SELECT @@sql_mode;' | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Nextcloud + env: + DB_PORT: 4444 + MARIADB_SHARDING: 1 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php + + - name: PHPUnit + run: composer run test:db -- --log-junit junit.xml ${{ matrix.coverage && '--coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + files: ./clover.db.xml + flags: phpunit-mariadb + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1 + with: + flags: phpunit-mariadb + + summary: + permissions: + contents: none + runs-on: ubuntu-latest-low + needs: [changes, phpunit-mariadb] + + if: always() + + name: phpunit-mariadb-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mariadb.result != 'success' }}; then exit 1; fi diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index 64b5a83477b6b..b682b0f28610a 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -8,13 +8,15 @@ use OCP\DB\ISchemaWrapper; use OCP\DB\Types; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Migration\IOutput; use OCP\Migration\SimpleMigrationStep; class Version13000Date20170718121200 extends SimpleMigrationStep { public function __construct( - private IDBConnection $connection, + private readonly IDBConnection $connection, + private readonly IConfig $config, ) { } @@ -229,7 +231,12 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op 'notnull' => false, 'length' => 255, ]); - $table->setPrimaryKey(['fileid']); + $maridbSharding = $this->config->getSystemValue('mariadbsharing'); + if ($maridbSharding && isset($maridbSharding['filecache']) && $maridbSharding['filecache'] === true && $this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_MARIADB) { + $table->setPrimaryKey(['fileid', 'storage']); + } else { + $table->setPrimaryKey(['fileid']); + } $table->addUniqueIndex(['storage', 'path_hash'], 'fs_storage_path_hash'); $table->addIndex(['parent', 'name'], 'fs_parent_name_hash'); $table->addIndex(['storage', 'mimetype'], 'fs_storage_mimetype'); @@ -1027,5 +1034,11 @@ public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array ->setParameter('userid', ($match[2] ?? '')); $insert->executeStatement(); } + + $maridbSharding = $this->config->getSystemValue('mariadbsharing'); + if ($maridbSharding && isset($maridbSharding['filecache']) && $maridbSharding['filecache'] === true + && $this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_MARIADB) { + $this->connection->prepare('ALTER TABLE *PREFIX*filecache PARTITION BY HASH(storage) PARTITIONS 8')->execute(); + } } } diff --git a/tests/preseed-config.php b/tests/preseed-config.php index 1fc98a4226ac0..648798b8a7a6c 100644 --- a/tests/preseed-config.php +++ b/tests/preseed-config.php @@ -117,3 +117,9 @@ ] ]; } + +if (getenv('MARIADB_SHARDING') == '1') { + $CONFIG['mariadbsharing'] = [ + 'filecache' => true, + ]; +}