diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58a8b9dca12..2ae66882420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: npm test - name: Image Embedder bundling run: ./scripts/embedder-deploy/build.sh - - uses: guardian/actions-riff-raff@v4.2.4 + - uses: guardian/actions-riff-raff@v4.3.3 if: "! github.event.pull_request.head.repo.fork" with: githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sbt-dependency-graph.yaml b/.github/workflows/sbt-dependency-graph.yaml index f9aad713eca..68729d8a7be 100644 --- a/.github/workflows/sbt-dependency-graph.yaml +++ b/.github/workflows/sbt-dependency-graph.yaml @@ -19,7 +19,7 @@ jobs: java-version: 17 - name: Install sbt id: sbt - uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + uses: sbt/setup-sbt@4ed7b7ec4bfa2074fe48554c09e341267397247c # v1.2.0 - name: Submit dependencies id: submit uses: scalacenter/sbt-dependency-submission@f43202114d7522a4b233e052f82c2eea8d658134 # v3.2.1 diff --git a/build.sbt b/build.sbt index dddedaa6e51..571b0bc9e53 100644 --- a/build.sbt +++ b/build.sbt @@ -76,8 +76,8 @@ Global / concurrentRestrictions := Seq( Tags.limitAll(12) ) -val awsSdkVersion = "1.12.470" -val awsSdkV2Version = "2.42.25" +val awsSdkVersion = "1.12.797" +val awsSdkV2Version = "2.44.13" val elastic4sVersion = "8.19.1" val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 67a6ee77825..0af9faa80f8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -16,7 +16,7 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { new ReceiveMessageRequest(queueUrl) .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .withAttributeNames(attributeNames: _*) + .withMessageSystemAttributeNames(attributeNames: _*) ).getMessages.asScala.headOption def deleteMessage(message: SQSMessage): Unit = diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala index ceb44bf61dd..f39a097786b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala @@ -13,7 +13,8 @@ class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConf ), url = string("es6.url"), shards = string("es6.shards").toInt, - replicas = string("es6.replicas").toInt + replicas = string("es6.replicas").toInt, + includeDenseVectorMappings = booleanOpt("es.includeDenseVectorMappings").getOrElse(true) ) private val persistenceIdentifier = string("persistence.identifier") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala index ee6ca13fbd4..6fbb23682f3 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala @@ -37,6 +37,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { def shards: Int def replicas: Int + def includeDenseVectorMappings: Boolean lazy val client = { logger.info("Connecting to Elastic 8: " + url) @@ -146,7 +147,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { "to remove the need for these: " + nonRecommendenedIndexSettingOverrides) createIndex(index). - mapping(Mappings.imageMapping). + mapping(Mappings.imageMapping(includeDenseVectorMappings)). analysis(IndexSettings.analysis). settings(nonRecommendenedIndexSettingOverrides). shards(shards). diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchConfig.scala index 8d27c303bf1..4b385f4e6a1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchConfig.scala @@ -1,3 +1,3 @@ package com.gu.mediaservice.lib.elasticsearch -case class ElasticSearchConfig(aliases: ElasticSearchAliases, url: String, shards: Int, replicas: Int) +case class ElasticSearchConfig(aliases: ElasticSearchAliases, url: String, shards: Int, replicas: Int, includeDenseVectorMappings: Boolean = true) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala index 8a87e9124db..f2fd652f869 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala @@ -8,7 +8,7 @@ import play.api.libs.json.{JsObject, Json} object Mappings { - val imageMapping: MappingDefinition = { + def imageMapping(includeDenseVectorMappings: Boolean = true): MappingDefinition = { // Non indexed fields stored as keywords can still participate in exists / has queries def filemetaDataStringsAsKeyword: DynamicTemplateRequest = { @@ -75,8 +75,7 @@ object Mappings { leasesMapping("leases"), collectionMapping("collections"), esInfoMapping("esInfo"), - embeddingMapping("embedding"), - ) + ) ++ (if (includeDenseVectorMappings) Seq(embeddingMapping("embedding")) else Seq.empty) ) } diff --git a/dev/oidc-provider/Dockerfile b/dev/oidc-provider/Dockerfile index 24ae5ae3a39..51efc744a10 100644 --- a/dev/oidc-provider/Dockerfile +++ b/dev/oidc-provider/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16 +FROM node:24 WORKDIR /usr/src/app diff --git a/dev/oidc-provider/app.js b/dev/oidc-provider/app.js index e4fd510a8b7..f2753772af2 100644 --- a/dev/oidc-provider/app.js +++ b/dev/oidc-provider/app.js @@ -2,7 +2,7 @@ import { findAccountFunc } from "./find-account.js"; import { makeProvider } from "./make-provider.js"; // relative path to the users file within the localstack box ( to /etc/grid/users.json) -import USER_JSON from "../../../etc/grid/users.json" assert { type: "json" }; +import USER_JSON from "../../../etc/grid/users.json" with { type: "json" }; const { DOMAIN, EMAIL_DOMAIN, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET } = process.env; diff --git a/dev/oidc-provider/package-lock.json b/dev/oidc-provider/package-lock.json index d14276de07e..5a792a9a130 100644 --- a/dev/oidc-provider/package-lock.json +++ b/dev/oidc-provider/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "oidc-provider": "^8.5.1" + "oidc-provider": "^9.8.3" }, "devDependencies": { "jest": "^29.7.0" @@ -959,18 +959,26 @@ } }, "node_modules/@koa/router": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/@koa/router/-/router-12.0.1.tgz", - "integrity": "sha512-ribfPYfHb+Uw3b27Eiw6NPqjhIhTpVFzEWLwyc/1Xp+DCdwRRyIlAUODX+9bPARF6aQtUu1+/PHzdNvRzcs/+Q==", + "version": "15.5.0", + "resolved": "https://registry.npmjs.org/@koa/router/-/router-15.5.0.tgz", + "integrity": "sha512-KSC0oG/5t6ITu5wqX4lJseA/dngoj14hEaohrLZEXtlUT2RRyJvwaJ0KV+5uQoaWrY3A8ClHOrBEU4g8dujn8Q==", + "license": "MIT", "dependencies": { - "debug": "^4.3.4", - "http-errors": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", "koa-compose": "^4.1.0", - "methods": "^1.1.2", - "path-to-regexp": "^6.2.1" + "path-to-regexp": "^8.4.2" }, "engines": { - "node": ">= 12" + "node": ">= 20" + }, + "peerDependencies": { + "koa": "^2.0.0 || ^3.0.0" + }, + "peerDependenciesMeta": { + "koa": { + "optional": false + } } }, "node_modules/@sinclair/typebox": { @@ -979,17 +987,6 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, - "node_modules/@sindresorhus/is": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", - "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", @@ -1008,17 +1005,6 @@ "@sinonjs/commons": "^3.0.0" } }, - "node_modules/@szmarczak/http-timer": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz", - "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==", - "dependencies": { - "defer-to-connect": "^2.0.1" - }, - "engines": { - "node": ">=14.16" - } - }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -1069,11 +1055,6 @@ "@types/node": "*" } }, - "node_modules/@types/http-cache-semantics": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==" - }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -1129,6 +1110,7 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" @@ -1137,6 +1119,27 @@ "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -1374,47 +1377,11 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/cache-content-type": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", - "integrity": "sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==", - "dependencies": { - "mime-types": "^2.1.18", - "ylru": "^1.2.0" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/cacheable-lookup": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", - "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", - "engines": { - "node": ">=14.16" - } - }, - "node_modules/cacheable-request": { - "version": "10.2.14", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz", - "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==", - "dependencies": { - "@types/http-cache-semantics": "^4.0.2", - "get-stream": "^6.0.1", - "http-cache-semantics": "^4.1.1", - "keyv": "^4.5.3", - "mimic-response": "^4.0.0", - "normalize-url": "^8.0.0", - "responselike": "^3.0.0" - }, - "engines": { - "node": ">=14.16" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1517,6 +1484,7 @@ "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" @@ -1553,20 +1521,23 @@ "dev": true }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "dependencies": { - "safe-buffer": "5.2.1" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -1581,6 +1552,7 @@ "version": "0.9.1", "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.9.1.tgz", "integrity": "sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==", + "license": "MIT", "dependencies": { "depd": "~2.0.0", "keygrip": "~1.1.0" @@ -1625,11 +1597,12 @@ } }, "node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -1640,31 +1613,6 @@ } } }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/decompress-response/node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/dedent": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", @@ -1682,7 +1630,8 @@ "node_modules/deep-equal": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==" + "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==", + "license": "MIT" }, "node_modules/deepmerge": { "version": "4.3.1", @@ -1693,23 +1642,17 @@ "node": ">=0.10.0" } }, - "node_modules/defer-to-connect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", - "engines": { - "node": ">=10" - } - }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT" }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -1718,6 +1661,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -1744,7 +1688,8 @@ "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" }, "node_modules/electron-to-chromium": { "version": "1.5.0", @@ -1771,9 +1716,10 @@ "dev": true }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -1799,7 +1745,8 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" }, "node_modules/escape-string-regexp": { "version": "2.0.0", @@ -1824,14 +1771,15 @@ } }, "node_modules/eta": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/eta/-/eta-3.4.0.tgz", - "integrity": "sha512-tCsc7WXTjrTx4ZjYLplcqrI3o4mYJ+Z6YspeuGL8tbt/hHoMchwBwtKfwM09svEY86iRapY93vUqQttcNuIO5Q==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/eta/-/eta-4.6.0.tgz", + "integrity": "sha512-lW6is4T1NFOYnmqGZIfvixqj7A7sSvScF+DN8EK6K58xI5MZ5UvYe0GjopxOXQtZvUn4eDdVuZ8XSoYWTMEKwA==", + "license": "MIT", "engines": { - "node": ">=6.0.0" + "node": ">=20" }, "funding": { - "url": "https://github.com/eta-dev/eta?sponsor=1" + "url": "https://github.com/bgub/eta?sponsor=1" } }, "node_modules/execa": { @@ -1922,18 +1870,11 @@ "node": ">=8" } }, - "node_modules/form-data-encoder": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", - "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==", - "engines": { - "node": ">= 14.17" - } - }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -1998,6 +1939,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, "engines": { "node": ">=10" }, @@ -2035,30 +1977,6 @@ "node": ">=4" } }, - "node_modules/got": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", - "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==", - "dependencies": { - "@sindresorhus/is": "^5.2.0", - "@szmarczak/http-timer": "^5.0.1", - "cacheable-lookup": "^7.0.0", - "cacheable-request": "^10.2.8", - "decompress-response": "^6.0.0", - "form-data-encoder": "^2.1.2", - "get-stream": "^6.0.1", - "http2-wrapper": "^2.1.10", - "lowercase-keys": "^3.0.0", - "p-cancelable": "^3.0.0", - "responselike": "^3.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sindresorhus/got?sponsor=1" - } - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -2074,31 +1992,6 @@ "node": ">=8" } }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -2121,6 +2014,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz", "integrity": "sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==", + "license": "MIT", "dependencies": { "deep-equal": "~1.0.1", "http-errors": "~1.8.0" @@ -2133,6 +2027,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -2141,6 +2036,7 @@ "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "license": "MIT", "dependencies": { "depd": "~1.1.2", "inherits": "2.0.4", @@ -2156,51 +2052,29 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/http-cache-semantics": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", - "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" - }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" - } - }, - "node_modules/http2-wrapper": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", - "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", - "dependencies": { - "quick-lru": "^5.1.1", - "resolve-alpn": "^1.2.0" - }, - "engines": { - "node": ">=10.19.0" - } - }, - "node_modules/http2-wrapper/node_modules/quick-lru": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", - "engines": { - "node": ">=10" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/human-signals": { @@ -2213,14 +2087,19 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/import-local": { @@ -2306,20 +2185,6 @@ "node": ">=6" } }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -2993,9 +2858,10 @@ } }, "node_modules/jose": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-5.6.3.tgz", - "integrity": "sha512-1Jh//hEEwMhNYPDDLwXHa2ePWgWiFNNUadVmguAAw2IJ6sj9mNxV5tGXJNqlMkJAybF6Lgw1mISDxTePP/187g==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/panva" } @@ -3020,9 +2886,10 @@ } }, "node_modules/jsesc": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", - "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", "bin": { "jsesc": "bin/jsesc" }, @@ -3030,11 +2897,6 @@ "node": ">=6" } }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" - }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -3057,6 +2919,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", "integrity": "sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==", + "license": "MIT", "dependencies": { "tsscmp": "1.0.6" }, @@ -3064,14 +2927,6 @@ "node": ">= 0.6" } }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dependencies": { - "json-buffer": "3.0.1" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -3082,86 +2937,39 @@ } }, "node_modules/koa": { - "version": "2.15.4", - "resolved": "https://registry.npmjs.org/koa/-/koa-2.15.4.tgz", - "integrity": "sha512-7fNBIdrU2PEgLljXoPWoyY4r1e+ToWCmzS/wwMPbUNs7X+5MMET1ObhJBlUkF5uZG9B6QhM2zS1TsH6adegkiQ==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/koa/-/koa-3.2.1.tgz", + "integrity": "sha512-e7IpWJrnanNUroVK2taAgMxoEZvHLXdQiNjeExSu/DEIWm83jaKGBgb7tLmu2rMYpA027qFB3iLR/k3AVpFRnA==", "license": "MIT", "dependencies": { - "accepts": "^1.3.5", - "cache-content-type": "^1.0.0", - "content-disposition": "~0.5.2", - "content-type": "^1.0.4", - "cookies": "~0.9.0", - "debug": "^4.3.2", + "accepts": "^1.3.8", + "content-disposition": "~1.0.1", + "content-type": "^1.0.5", + "cookies": "~0.9.1", "delegates": "^1.0.0", - "depd": "^2.0.0", - "destroy": "^1.0.4", - "encodeurl": "^1.0.2", + "destroy": "^1.2.0", + "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "fresh": "~0.5.2", - "http-assert": "^1.3.0", - "http-errors": "^1.6.3", - "is-generator-function": "^1.0.7", + "http-assert": "^1.5.0", + "http-errors": "^2.0.0", "koa-compose": "^4.1.0", - "koa-convert": "^2.0.0", - "on-finished": "^2.3.0", - "only": "~0.0.2", - "parseurl": "^1.3.2", - "statuses": "^1.5.0", - "type-is": "^1.6.16", + "mime-types": "^3.0.1", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1", + "type-is": "^2.0.1", "vary": "^1.1.2" }, "engines": { - "node": "^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4" + "node": ">= 18" } }, "node_modules/koa-compose": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", - "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==" - }, - "node_modules/koa-convert": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz", - "integrity": "sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==", - "dependencies": { - "co": "^4.6.0", - "koa-compose": "^4.1.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/koa/node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/koa/node_modules/http-errors/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/koa/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "engines": { - "node": ">= 0.6" - } + "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==", + "license": "MIT" }, "node_modules/leven": { "version": "3.1.0", @@ -3190,17 +2998,6 @@ "node": ">=8" } }, - "node_modules/lowercase-keys": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", - "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -3247,11 +3044,12 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/merge-stream": { @@ -3260,14 +3058,6 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -3282,22 +3072,28 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/mimic-fn": { @@ -3309,17 +3105,6 @@ "node": ">=6" } }, - "node_modules/mimic-response": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", - "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3343,20 +3128,22 @@ } }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/nanoid": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz", - "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==", + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz", + "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -3374,6 +3161,7 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -3399,17 +3187,6 @@ "node": ">=0.10.0" } }, - "node_modules/normalize-url": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz", - "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -3422,49 +3199,32 @@ "node": ">=8" } }, - "node_modules/object-hash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", - "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "engines": { - "node": ">= 6" - } - }, "node_modules/oidc-provider": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-8.5.1.tgz", - "integrity": "sha512-Bm3EyxN68/KS76IlciJ3+4pnVtfdRWL+NghWpIF0XQbiRT1gzc6Qf/cyFmpL9yieko/jXYZ/uLHUv77jD00qww==", + "version": "9.8.3", + "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-9.8.3.tgz", + "integrity": "sha512-YkchaAyVAZbsn/l7IQhcEMdeDL3lwSo/PNUtnsXSqPqT7EG8DRko0EAWzHd/n9VfCtKVkxGjYOY4h4UwFcWnUA==", + "license": "MIT", "dependencies": { "@koa/cors": "^5.0.0", - "@koa/router": "^12.0.1", - "debug": "^4.3.5", - "eta": "^3.4.0", - "got": "^13.0.0", - "jose": "^5.6.2", - "jsesc": "^3.0.2", - "koa": "^2.15.3", - "nanoid": "^5.0.7", - "object-hash": "^3.0.0", - "oidc-token-hash": "^5.0.3", - "quick-lru": "^7.0.0", - "raw-body": "^2.5.2" + "@koa/router": "^15.4.0", + "debug": "^4.4.3", + "eta": "^4.5.1", + "jose": "^6.2.2", + "jsesc": "^3.1.0", + "koa": "^3.2.0", + "nanoid": "^5.1.7", + "quick-lru": "^7.3.0", + "raw-body": "^3.0.2" }, "funding": { "url": "https://github.com/sponsors/panva" } }, - "node_modules/oidc-token-hash": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz", - "integrity": "sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw==", - "engines": { - "node": "^10.13.0 || >=12.0.0" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", "dependencies": { "ee-first": "1.1.1" }, @@ -3496,19 +3256,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/only": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz", - "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==" - }, - "node_modules/p-cancelable": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", - "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==", - "engines": { - "node": ">=12.20" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3582,6 +3329,7 @@ "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -3620,9 +3368,14 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/picocolors": { "version": "1.0.1", @@ -3719,9 +3472,10 @@ ] }, "node_modules/quick-lru": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.0.tgz", - "integrity": "sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg==", + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==", + "license": "MIT", "engines": { "node": ">=18" }, @@ -3730,17 +3484,18 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" } }, "node_modules/react-is": { @@ -3775,11 +3530,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/resolve-alpn": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" - }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -3810,43 +3560,11 @@ "node": ">=10" } }, - "node_modules/responselike": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", - "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", - "dependencies": { - "lowercase-keys": "^3.0.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" }, "node_modules/semver": { "version": "6.3.1", @@ -3860,7 +3578,8 @@ "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" }, "node_modules/shebang-command": { "version": "2.0.0", @@ -3942,9 +3661,10 @@ } }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -4087,6 +3807,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", "engines": { "node": ">=0.6" } @@ -4095,6 +3816,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", "integrity": "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==", + "license": "MIT", "engines": { "node": ">=0.6.x" } @@ -4121,21 +3843,41 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -4294,14 +4036,6 @@ "node": ">=12" } }, - "node_modules/ylru": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.4.0.tgz", - "integrity": "sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -5037,15 +4771,14 @@ } }, "@koa/router": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/@koa/router/-/router-12.0.1.tgz", - "integrity": "sha512-ribfPYfHb+Uw3b27Eiw6NPqjhIhTpVFzEWLwyc/1Xp+DCdwRRyIlAUODX+9bPARF6aQtUu1+/PHzdNvRzcs/+Q==", + "version": "15.5.0", + "resolved": "https://registry.npmjs.org/@koa/router/-/router-15.5.0.tgz", + "integrity": "sha512-KSC0oG/5t6ITu5wqX4lJseA/dngoj14hEaohrLZEXtlUT2RRyJvwaJ0KV+5uQoaWrY3A8ClHOrBEU4g8dujn8Q==", "requires": { - "debug": "^4.3.4", - "http-errors": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", "koa-compose": "^4.1.0", - "methods": "^1.1.2", - "path-to-regexp": "^6.2.1" + "path-to-regexp": "^8.4.2" } }, "@sinclair/typebox": { @@ -5054,11 +4787,6 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, - "@sindresorhus/is": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", - "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==" - }, "@sinonjs/commons": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", @@ -5077,14 +4805,6 @@ "@sinonjs/commons": "^3.0.0" } }, - "@szmarczak/http-timer": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz", - "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==", - "requires": { - "defer-to-connect": "^2.0.1" - } - }, "@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -5135,11 +4855,6 @@ "@types/node": "*" } }, - "@types/http-cache-semantics": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==" - }, "@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -5198,6 +4913,21 @@ "requires": { "mime-types": "~2.1.34", "negotiator": "0.6.3" + }, + "dependencies": { + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + } } }, "ansi-escapes": { @@ -5375,34 +5105,6 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" }, - "cache-content-type": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", - "integrity": "sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==", - "requires": { - "mime-types": "^2.1.18", - "ylru": "^1.2.0" - } - }, - "cacheable-lookup": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", - "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==" - }, - "cacheable-request": { - "version": "10.2.14", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz", - "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==", - "requires": { - "@types/http-cache-semantics": "^4.0.2", - "get-stream": "^6.0.1", - "http-cache-semantics": "^4.1.1", - "keyv": "^4.5.3", - "mimic-response": "^4.0.0", - "normalize-url": "^8.0.0", - "responselike": "^3.0.0" - } - }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -5463,7 +5165,8 @@ "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==" + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true }, "collect-v8-coverage": { "version": "1.0.2", @@ -5493,12 +5196,9 @@ "dev": true }, "content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "requires": { - "safe-buffer": "5.2.1" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==" }, "content-type": { "version": "1.0.5", @@ -5547,26 +5247,11 @@ } }, "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { - "mimic-response": "^3.1.0" - }, - "dependencies": { - "mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" - } + "ms": "^2.1.3" } }, "dedent": { @@ -5587,11 +5272,6 @@ "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "dev": true }, - "defer-to-connect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==" - }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -5643,9 +5323,9 @@ "dev": true }, "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==" }, "error-ex": { "version": "1.3.2", @@ -5680,9 +5360,9 @@ "dev": true }, "eta": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/eta/-/eta-3.4.0.tgz", - "integrity": "sha512-tCsc7WXTjrTx4ZjYLplcqrI3o4mYJ+Z6YspeuGL8tbt/hHoMchwBwtKfwM09svEY86iRapY93vUqQttcNuIO5Q==" + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/eta/-/eta-4.6.0.tgz", + "integrity": "sha512-lW6is4T1NFOYnmqGZIfvixqj7A7sSvScF+DN8EK6K58xI5MZ5UvYe0GjopxOXQtZvUn4eDdVuZ8XSoYWTMEKwA==" }, "execa": { "version": "5.1.1", @@ -5754,11 +5434,6 @@ "path-exists": "^4.0.0" } }, - "form-data-encoder": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", - "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==" - }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -5804,7 +5479,8 @@ "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true }, "glob": { "version": "7.2.3", @@ -5826,24 +5502,6 @@ "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, - "got": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", - "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==", - "requires": { - "@sindresorhus/is": "^5.2.0", - "@szmarczak/http-timer": "^5.0.1", - "cacheable-lookup": "^7.0.0", - "cacheable-request": "^10.2.8", - "decompress-response": "^6.0.0", - "form-data-encoder": "^2.1.2", - "get-stream": "^6.0.1", - "http2-wrapper": "^2.1.10", - "lowercase-keys": "^3.0.0", - "p-cancelable": "^3.0.0", - "responselike": "^3.0.0" - } - }, "graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -5856,19 +5514,6 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" - }, - "has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "requires": { - "has-symbols": "^1.0.3" - } - }, "hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -5917,37 +5562,16 @@ } } }, - "http-cache-semantics": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", - "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" - }, "http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", - "requires": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - } - }, - "http2-wrapper": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", - "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "requires": { - "quick-lru": "^5.1.1", - "resolve-alpn": "^1.2.0" - }, - "dependencies": { - "quick-lru": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==" - } + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" } }, "human-signals": { @@ -5957,11 +5581,11 @@ "dev": true }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "requires": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "import-local": { @@ -6022,14 +5646,6 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true }, - "is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "requires": { - "has-tostringtag": "^1.0.0" - } - }, "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -6542,9 +6158,9 @@ } }, "jose": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-5.6.3.tgz", - "integrity": "sha512-1Jh//hEEwMhNYPDDLwXHa2ePWgWiFNNUadVmguAAw2IJ6sj9mNxV5tGXJNqlMkJAybF6Lgw1mISDxTePP/187g==" + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==" }, "js-tokens": { "version": "4.0.0", @@ -6563,14 +6179,9 @@ } }, "jsesc": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", - "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==" - }, - "json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==" }, "json-parse-even-better-errors": { "version": "2.3.1", @@ -6592,14 +6203,6 @@ "tsscmp": "1.0.6" } }, - "keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "requires": { - "json-buffer": "3.0.1" - } - }, "kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -6607,59 +6210,28 @@ "dev": true }, "koa": { - "version": "2.15.4", - "resolved": "https://registry.npmjs.org/koa/-/koa-2.15.4.tgz", - "integrity": "sha512-7fNBIdrU2PEgLljXoPWoyY4r1e+ToWCmzS/wwMPbUNs7X+5MMET1ObhJBlUkF5uZG9B6QhM2zS1TsH6adegkiQ==", - "requires": { - "accepts": "^1.3.5", - "cache-content-type": "^1.0.0", - "content-disposition": "~0.5.2", - "content-type": "^1.0.4", - "cookies": "~0.9.0", - "debug": "^4.3.2", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/koa/-/koa-3.2.1.tgz", + "integrity": "sha512-e7IpWJrnanNUroVK2taAgMxoEZvHLXdQiNjeExSu/DEIWm83jaKGBgb7tLmu2rMYpA027qFB3iLR/k3AVpFRnA==", + "requires": { + "accepts": "^1.3.8", + "content-disposition": "~1.0.1", + "content-type": "^1.0.5", + "cookies": "~0.9.1", "delegates": "^1.0.0", - "depd": "^2.0.0", - "destroy": "^1.0.4", - "encodeurl": "^1.0.2", + "destroy": "^1.2.0", + "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "fresh": "~0.5.2", - "http-assert": "^1.3.0", - "http-errors": "^1.6.3", - "is-generator-function": "^1.0.7", + "http-assert": "^1.5.0", + "http-errors": "^2.0.0", "koa-compose": "^4.1.0", - "koa-convert": "^2.0.0", - "on-finished": "^2.3.0", - "only": "~0.0.2", - "parseurl": "^1.3.2", - "statuses": "^1.5.0", - "type-is": "^1.6.16", + "mime-types": "^3.0.1", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1", + "type-is": "^2.0.1", "vary": "^1.1.2" - }, - "dependencies": { - "http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "dependencies": { - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" - } - } - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" - } } }, "koa-compose": { @@ -6667,15 +6239,6 @@ "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==" }, - "koa-convert": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz", - "integrity": "sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==", - "requires": { - "co": "^4.6.0", - "koa-compose": "^4.1.0" - } - }, "leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -6697,11 +6260,6 @@ "p-locate": "^4.1.0" } }, - "lowercase-keys": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", - "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==" - }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -6738,9 +6296,9 @@ } }, "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==" }, "merge-stream": { "version": "2.0.0", @@ -6748,11 +6306,6 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" - }, "micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -6764,16 +6317,16 @@ } }, "mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==" }, "mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "requires": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" } }, "mimic-fn": { @@ -6782,11 +6335,6 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, - "mimic-response": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", - "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==" - }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -6809,14 +6357,14 @@ } }, "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nanoid": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz", - "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==" + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz", + "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg==" }, "natural-compare": { "version": "1.4.0", @@ -6847,11 +6395,6 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, - "normalize-url": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz", - "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==" - }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6861,35 +6404,22 @@ "path-key": "^3.0.0" } }, - "object-hash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", - "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" - }, "oidc-provider": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-8.5.1.tgz", - "integrity": "sha512-Bm3EyxN68/KS76IlciJ3+4pnVtfdRWL+NghWpIF0XQbiRT1gzc6Qf/cyFmpL9yieko/jXYZ/uLHUv77jD00qww==", + "version": "9.8.3", + "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-9.8.3.tgz", + "integrity": "sha512-YkchaAyVAZbsn/l7IQhcEMdeDL3lwSo/PNUtnsXSqPqT7EG8DRko0EAWzHd/n9VfCtKVkxGjYOY4h4UwFcWnUA==", "requires": { "@koa/cors": "^5.0.0", - "@koa/router": "^12.0.1", - "debug": "^4.3.5", - "eta": "^3.4.0", - "got": "^13.0.0", - "jose": "^5.6.2", - "jsesc": "^3.0.2", - "koa": "^2.15.3", - "nanoid": "^5.0.7", - "object-hash": "^3.0.0", - "oidc-token-hash": "^5.0.3", - "quick-lru": "^7.0.0", - "raw-body": "^2.5.2" - } - }, - "oidc-token-hash": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz", - "integrity": "sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw==" + "@koa/router": "^15.4.0", + "debug": "^4.4.3", + "eta": "^4.5.1", + "jose": "^6.2.2", + "jsesc": "^3.1.0", + "koa": "^3.2.0", + "nanoid": "^5.1.7", + "quick-lru": "^7.3.0", + "raw-body": "^3.0.2" + } }, "on-finished": { "version": "2.4.1", @@ -6917,16 +6447,6 @@ "mimic-fn": "^2.1.0" } }, - "only": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz", - "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==" - }, - "p-cancelable": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", - "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -7004,9 +6524,9 @@ "dev": true }, "path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==" }, "picocolors": { "version": "1.0.1", @@ -7071,19 +6591,19 @@ "dev": true }, "quick-lru": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.0.tgz", - "integrity": "sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg==" + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==" }, "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" } }, "react-is": { @@ -7109,11 +6629,6 @@ "supports-preserve-symlinks-flag": "^1.0.0" } }, - "resolve-alpn": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" - }, "resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -7135,19 +6650,6 @@ "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", "dev": true }, - "responselike": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", - "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", - "requires": { - "lowercase-keys": "^3.0.0" - } - }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" - }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -7229,9 +6731,9 @@ } }, "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" }, "string-length": { "version": "4.0.2", @@ -7351,12 +6853,20 @@ "dev": true }, "type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "dependencies": { + "content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==" + } } }, "unpipe": { @@ -7468,11 +6978,6 @@ "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", "dev": true }, - "ylru": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.4.0.tgz", - "integrity": "sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==" - }, "yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/dev/oidc-provider/package.json b/dev/oidc-provider/package.json index f9e00a8d852..52b22cabc14 100644 --- a/dev/oidc-provider/package.json +++ b/dev/oidc-provider/package.json @@ -9,7 +9,7 @@ "author": "", "license": "ISC", "dependencies": { - "oidc-provider": "^8.5.1" + "oidc-provider": "^9.8.3" }, "devDependencies": { "jest": "^29.7.0" diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index dc208300b46..2424dac951f 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -146,7 +146,7 @@ function getKahunaConfig(config){ |]`; const permissionsConfig = stripMargin` - |usePermissionsFilter=true + |usePermissionsFilter=false |permissionsDefault="allPermissions" `; @@ -191,6 +191,7 @@ function getMediaApiConfig(config) { |es6.url="${config.es6.url}" |es6.shards=${config.es6.shards} |es6.replicas=${config.es6.replicas} + |es.includeDenseVectorMappings=true |quota.store.key="rcs-quota.json" |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}" |search.fuzziness={ @@ -225,6 +226,7 @@ function getThrallConfig(config) { |es6.url="${config.es6.url}" |es6.shards=${config.es6.shards} |es6.replicas=${config.es6.replicas} + |es.includeDenseVectorMappings=true |metrics.request.enabled=false |`; } diff --git a/kahuna/public/js/components/gr-chips/gr-chips.js b/kahuna/public/js/components/gr-chips/gr-chips.js index 5911eee453e..d3db7528aad 100644 --- a/kahuna/public/js/components/gr-chips/gr-chips.js +++ b/kahuna/public/js/components/gr-chips/gr-chips.js @@ -27,7 +27,7 @@ grChips.controller('grChipsCtrl', ['$scope', function($scope) { $grChipsCtrl.configureNgModel = function(ngModelCtrl, onChangeExpr, autoCompleteExpr, validKeysExpr, autofocus, placeholder) { $grChipsCtrl.defaultPlaceholder = placeholder || 'Search for images... (type + for advanced search)'; - $grChipsCtrl.aiSearchPlaceholder = 'Search for conceptual images using AI search... (no filters available)'; + $grChipsCtrl.aiSearchPlaceholder = 'Search for conceptual images using AI search...'; $grChipsCtrl.onChange = () => onChangeExpr($scope, {$chips: $grChipsCtrl.items}); diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index 8cfde8b1157..a01d32be66f 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -30,7 +30,7 @@ info_outline -
+
diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 1268fd9a9e2..6721bf53ba5 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -1,7 +1,6 @@ import angular from 'angular'; import Rx from 'rx'; import moment from 'moment'; - import '../services/scroll-position'; import '../services/panel'; import '../util/async'; diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index 2202bf95345..dd7a1e99618 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -37,6 +37,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) val elasticSearchHealthCheck = new ElasticSearchHealthCheck(controllerComponents, elasticSearch) val healthcheckController = new Management(controllerComponents, buildInfo) + val configurationController = new ConfigurationController(controllerComponents) val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient) override val router = new Routes( @@ -45,6 +46,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M suggestionController, aggController, usageController, + configurationController, elasticSearchHealthCheck, healthcheckController, InnerServiceStatusCheckController diff --git a/media-api/app/controllers/ConfigurationController.scala b/media-api/app/controllers/ConfigurationController.scala new file mode 100644 index 00000000000..99fa172f5ab --- /dev/null +++ b/media-api/app/controllers/ConfigurationController.scala @@ -0,0 +1,11 @@ +package controllers + +import lib.crops.CropOption +import play.api.libs.json.Json +import play.api.mvc.{BaseController, ControllerComponents} + +class ConfigurationController(override val controllerComponents: ControllerComponents) extends BaseController { + // The contents of this endpoint is not sensitive, so to avoid unnecessary complexity we have not + // put authentication on this + def cropVariations = Action { _ => Ok(Json.toJson(CropOption.supported)) } +} diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 5b32930b4a6..12c99aaeeae 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -16,6 +16,7 @@ import com.gu.mediaservice.lib.play.RequestLoggingFilter import com.gu.mediaservice.model._ import com.gu.mediaservice.syntax.MessageSubjects import com.gu.mediaservice.{GridClient, JsonDiff} +import com.sksamuel.elastic4s.requests.searches.queries.Query import lib._ import lib.elasticsearch._ import org.apache.http.entity.ContentType @@ -575,16 +576,26 @@ class MediaApi( val canViewDeletedImages = _searchParams.query.contains("is:deleted") && !hasDeletePermission sealed trait AiSearchMode - case class TextSearch(query: String) extends AiSearchMode + case object TextSearch extends AiSearchMode case class SimilarSearch(imageId: String) extends AiSearchMode - def parseAiSearchMode(query: String): AiSearchMode = { - query - .split("\\s+") - .find(_.startsWith("similar:")) - .flatMap(_.split(":", 2).lift(1).filter(_.nonEmpty)) - .map(SimilarSearch) - .getOrElse(TextSearch(query)) + def parseAiSearchMode(params: SearchParams): AiSearchMode = + params.aiQueryParts.similarImageId.map(SimilarSearch).getOrElse(TextSearch) + + def buildAiFilter(params: SearchParams): Option[Query] = { + val filterConditions = params.aiQueryParts.filterConditions + + val chipFilterOpt = + if (filterConditions.nonEmpty) Some(elasticSearch.queryBuilder.makeQuery(filterConditions)) + else None + + val requestFilterOpt = elasticSearch.queryBuilder.buildFilterOpt( + params, + elasticSearch.searchFilters, + elasticSearch.syndicationFilter + ) + + elasticSearch.searchFilters.filterAndFilter(chipFilterOpt, requestFilterOpt) } def emptyAiSearchResponse = @@ -601,7 +612,9 @@ class MediaApi( ) } - def semanticSearchByImage(imageId: String, k: Int): Future[SearchResults] = { + def semanticSearchByImage(imageId: String, k: Int, params: SearchParams): Future[SearchResults] = { + val filterOpt = buildAiFilter(params) + for { maybeImage <- elasticSearch.getImageById(imageId) maybeEmbedding = maybeImage @@ -612,52 +625,68 @@ class MediaApi( searchResults <- maybeEmbedding match { // If we have an embedding, perform the KNN search. If not, return an empty result set. case Some(embedding) => - elasticSearch.knnSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100)) + elasticSearch.knnSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) case None => Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } } yield searchResults } - def semanticSearchByText(query: String, k: Int, vecWeight: Option[Double]): Future[SearchResults] = { - // Normalise key so that "Dogs" and "dogs " share a cache entry. - val cacheKey = query.trim.toLowerCase + def semanticSearchByText(k: Int, params: SearchParams): Future[SearchResults] = { + // Separate the chips from the main query text + // So that we can embed just the query text + params.aiQueryParts.semanticQuery match { + case None => + logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + case Some(semanticQuery) => + // Normalise key so that "Dogs" and "dogs " share a cache entry. + val cacheKey = semanticQuery.trim.toLowerCase + val markers = logMarker ++ Map("query" -> semanticQuery) + + if (embeddingCache.getIfPresent(cacheKey).isDefined) { + logger.info(markers, s"AI search embedding cache hit query=$semanticQuery") + } else { + logger.info(markers, s"AI search embedding cache miss query=$semanticQuery") + } - val markers = logMarker ++ Map("query" -> query) + val weight = params.vecWeight.getOrElse(1.0) - if (embeddingCache.getIfPresent(cacheKey).isDefined) { - logger.info(markers, s"AI search embedding cache hit query=$query") - } else { - logger.info(markers, s"AI search embedding cache miss query=$query") - } + // cache.get(key) is atomic: if two requests race on the same key, only one + // load fires and both callers receive the same Future. + val embeddingFuture = embeddingCache.get(cacheKey) - val weight = vecWeight.getOrElse(1.0) + val filterOpt = buildAiFilter(params) - // cache.get(key) is atomic: if two requests race on the same key, only one - // load fires and both callers receive the same Future. - val embeddingFuture = embeddingCache.get(cacheKey) - - logger.info(markers, s"vecWeight for query '$query' is $weight") - for { - embedding <- embeddingFuture - searchResults <- elasticSearch.hybridSearch( - query = query, - queryEmbedding = embedding, - k = k, - numCandidates = Math.max(k * 2, 100), - vecWeight = weight, - ) - } yield searchResults + logger.info(markers, s"vecWeight for query '$semanticQuery' is $weight") + for { + embedding <- embeddingFuture + searchResults <- elasticSearch.hybridSearch( + query = semanticQuery, + queryEmbedding = embedding, + k = k, + numCandidates = Math.max(k * 2, 100), + vecWeight = weight, + filterOpt = filterOpt + ) + } yield searchResults + } } - def performAiSearchAndRespond(query: String, vecWeight: Option[Double]): Future[Result] = { - val k = config.aiSearchResultLimit - val searchResultsFuture = parseAiSearchMode(query) match { - case SimilarSearch(imageId) => semanticSearchByImage(imageId, k) - case TextSearch(textQuery) => semanticSearchByText(textQuery, k, vecWeight) - } + def performAiSearchAndRespond(params: SearchParams): Future[Result] = { + params.aiQueryParts.validationError match { + case Left(errorMessage) => + logger.info(logMarker, s"Invalid AI search query: $errorMessage") + Future.successful(respondError(UnprocessableEntity, "invalid-ai-search", errorMessage)) + case scala.util.Right(_) => + val k = config.aiSearchResultLimit + val searchResultsFuture = parseAiSearchMode(params) match { + case SimilarSearch(imageId) => semanticSearchByImage(imageId, k, params) + case TextSearch => semanticSearchByText(k, params) + } - searchResultsFuture.map(aiSearchResponseFromResults) + searchResultsFuture.map(aiSearchResponseFromResults) + } } if (_searchParams.useAISearch.contains(true)) { @@ -666,7 +695,7 @@ class MediaApi( case _ if _searchParams.length == 0 => emptyAiSearchResponse case Some(q) if !q.isBlank => - performAiSearchAndRespond(q, _searchParams.vecWeight) + performAiSearchAndRespond(_searchParams) // Empty queries do not make sense for AI search as we can // only rank results once we have a meaningful vector to compare with. // So return 0 results if the query was empty. diff --git a/media-api/app/lib/crops/CropOption.scala b/media-api/app/lib/crops/CropOption.scala new file mode 100644 index 00000000000..b041c120a97 --- /dev/null +++ b/media-api/app/lib/crops/CropOption.scala @@ -0,0 +1,15 @@ +package lib.crops + +import play.api.libs.json.{Json, OFormat} + +case class CropOption(key: String, ratio: String, ratioString: String) + +object CropOption { + implicit val jf: OFormat[CropOption] = Json.format[CropOption] + val supported = List( + CropOption("landscape", "5 / 4", "5:4"), + CropOption("portrait", "4 / 5", "4:5"), + CropOption("video", "16 / 9", "16:9"), + CropOption("square", "1", "1:1"), + ) +} diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 175fe3d63c6..0d890ba598a 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -75,6 +75,7 @@ class ElasticSearch( lazy val url = elasticConfig.url lazy val shards = elasticConfig.shards lazy val replicas = elasticConfig.replicas + lazy val includeDenseVectorMappings = elasticConfig.includeDenseVectorMappings private val SearchQueryTimeout = FiniteDuration(10, TimeUnit.SECONDS) // there is 15 seconds timeout set on cluster level as well @@ -178,20 +179,25 @@ class ElasticSearch( } } - def knnSearch(queryEmbedding: List[Float], k: Int, numCandidates: Int) + def knnSearch(queryEmbedding: List[Float], k: Int, numCandidates: Int, filterOpt: Option[Query]) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - val knn = Knn("embedding.cohereEmbedV4.image") + if (!includeDenseVectorMappings) { + logger.warn(logMarker, "knnSearch called but includeDenseVectorMappings=false, returning empty results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } else { + val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) .queryVector(queryEmbedding.map(_.toDouble)) .k(k) .numCandidates(numCandidates) - val searchRequest = ElasticDsl.search(imagesCurrentAlias) - .knn(knn) - .size(k) + val searchRequest = ElasticDsl.search(imagesCurrentAlias) + .knn(knn) + .size(k) - executeAndLog(withSearchQueryTimeout(searchRequest), "knn search").map { r => - val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + executeAndLog(withSearchQueryTimeout(searchRequest), "knn search").map { r => + val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } } } @@ -211,9 +217,12 @@ class ElasticSearch( // than cosine similarity (knn). So we get the max BM25 score for the query and use that to calculate // the scaling factor for the lexical part of the query, so that BM25 and knn scores are both between 0-1 scale // and can be effectively combined in a hybrid query. - private def fetchMaxBm25Score(query: String)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Double] = { + private def fetchMaxBm25Score(query: String, filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Double] = { + val baseQuery = createMultiMatchQuery(query) + val filteredQuery = filterOpt.map(filter => boolQuery().must(baseQuery).filter(filter)).getOrElse(baseQuery) + val maxScoreRequest = ElasticDsl.search(imagesCurrentAlias) - .query(createMultiMatchQuery(query)) + .query(filteredQuery) executeAndLog(withSearchQueryTimeout(maxScoreRequest), "max BM25 score").map { r => logger.info(logMarker, s"Max BM25 score for query '$query' is ${r.result.hits.maxScore}") @@ -227,9 +236,10 @@ class ElasticSearch( k: Int, numCandidates: Int, vecWeight: Double, - maxScore: Double + maxScore: Double, + filterOpt: Option[Query] )(implicit logMarker: LogMarker): SearchRequest = { - val knn = Knn("embedding.cohereEmbedV4.image") + val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) .queryVector(queryEmbedding) .k(k) .numCandidates(numCandidates) @@ -242,9 +252,9 @@ class ElasticSearch( // BM25 score to bring it to the same range as the cosine similarity. val scalingFactor = if (maxScore > 0.0) 1.0 / maxScore else 1.0 - // We want to apply only one boost if we can help it, so we scale the - // multi_match boost to be in line with the max_score and the desired - // lexical_weight/vec_weight balance + // We want to apply only one boost if we can help it, so we scale the + // multi_match boost to be in line with the max_score and the desired + // lexical_weight/vec_weight balance val multiMatchBoost = if (vecWeight > 0.0) (lexicalWeight / vecWeight) * scalingFactor else 1.0 logger.info(logMarker, s"Scaling factor for BM25 score is $scalingFactor, multi-match boost is $multiMatchBoost") @@ -252,7 +262,7 @@ class ElasticSearch( val multiMatchQuery = createMultiMatchQuery(query, boost = Some(multiMatchBoost)) ElasticDsl.search(imagesCurrentAlias) - .bool(BoolQuery().should(Seq(multiMatchQuery, knn))) + .bool(BoolQuery().should(Seq(multiMatchQuery, knn)).filter(filterOpt)) .size(k) } @@ -262,103 +272,32 @@ class ElasticSearch( k: Int, numCandidates: Int, vecWeight: Double, + filterOpt: Option[Query] )( implicit ex: ExecutionContext, logMarker: LogMarker ): Future[SearchResults] = { - val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) - - for { - maxScore <- fetchMaxBm25Score(query) - searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore) - result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") - } yield { - val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + if (!includeDenseVectorMappings) { + logger.warn(logMarker, "hybridSearch called but includeDenseVectorMappings=false, returning empty results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } else { + val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) + + for { + maxScore <- fetchMaxBm25Score(query, filterOpt) + searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) + result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") + } yield { + val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } } } def search(params: SearchParams)(implicit ex: ExecutionContext, request: AuthenticatedRequest[AnyContent, Principal], logMarker: LogMarker = MarkerMap()): Future[SearchResults] = { val query: Query = queryBuilder.makeQuery(params.structuredQuery) - val uploadTimeFilter = filters.date("uploadTime", params.since, params.until) - val lastModTimeFilter = filters.date("lastModified", params.modifiedSince, params.modifiedUntil) - val takenTimeFilter = filters.date("metadata.dateTaken", params.takenSince, params.takenUntil) - // we only inject filters if there are actual date parameters - val dateFilterList = List(uploadTimeFilter, lastModTimeFilter, takenTimeFilter).flatten.toNel - val dateFilter = dateFilterList.map(dateFilters => filters.and(dateFilters.list.toList: _*)) - - val idsFilter = params.ids.map(filters.ids) - val labelFilter = params.labels.toNel.map(filters.terms("labels", _)) - val metadataFilter = params.hasMetadata.map(metadataField).toNel.map(filters.exists) - val archivedFilter = params.archived.map(filters.existsOrMissing(editsField("archived"), _)) - val hasExports = params.hasExports.map(filters.existsOrMissing("exports", _)) - val hasIdentifier = params.hasIdentifier.map(idName => filters.exists(NonEmptyList(identifierField(idName)))) - val missingIdentifier = params.missingIdentifier.map(idName => filters.missing(NonEmptyList(identifierField(idName)))) - val uploadedByFilter = params.uploadedBy.map(uploadedBy => filters.terms("uploadedBy", NonEmptyList(uploadedBy))) - val simpleCostFilter = params.free.flatMap(free => if (free) searchFilters.freeFilter else searchFilters.nonFreeFilter) - val costFilter = params.payType match { - case Some(PayType.Free) => searchFilters.freeFilter - case Some(PayType.MaybeFree) => searchFilters.maybeFreeFilter - case Some(PayType.Pay) => searchFilters.nonFreeFilter - case _ => None - } - - val printUsageFilter = params.printUsageFilters.map(searchFilters.printUsageFilters) - - val hasRightsCategory = params.hasRightsCategory.filter(_ == true).map(_ => searchFilters.hasRightsCategoryFilter) - - val validityFilter = params.valid.map(valid => if (valid) searchFilters.validFilter else searchFilters.invalidFilter) - - val persistFilter = params.persisted map { - case true => searchFilters.persistedFilter - case false => searchFilters.nonPersistedFilter - } - - val usageFilter: Iterable[Query] = - params.usageStatus.toNel.map(status => filters.terms("usagesStatus", status.map(_.toString))).toOption ++ - params.usagePlatform.toNel.map(filters.terms("usagesPlatform", _)).toOption - - val syndicationStatusFilter = params.syndicationStatus.map(status => syndicationFilter.statusFilter(status)) - - // Port of special case code in elastic1 sorts. Using the dateAddedToCollection sort implies an additional filter for reasons unknown - val dateAddedToCollectionFilter = { - params.orderBy match { - case Some("dateAddedToCollection") => { - val pathHierarchyOpt = params.structuredQuery.flatMap { - case Match(HierarchyField, Phrase(value)) => Some(value) - case _ => None - }.headOption - - pathHierarchyOpt.map { pathHierarchy => - termQuery("collections.pathHierarchy", pathHierarchy) - } - } - case _ => None - } - } - - val filterOpt = ( - metadataFilter.toOption.toList - ++ persistFilter - ++ labelFilter.toOption - ++ archivedFilter - ++ uploadedByFilter - ++ idsFilter - ++ validityFilter - ++ simpleCostFilter - ++ costFilter - ++ hasExports - ++ hasIdentifier - ++ missingIdentifier - ++ dateFilter.toOption - ++ usageFilter - ++ hasRightsCategory - ++ searchFilters.tierFilter(params.tier) - ++ syndicationStatusFilter - ++ dateAddedToCollectionFilter - ++ printUsageFilter - ).toNel.map(filter => filter.list.toList.reduceLeft(filters.and(_, _))).toOption + val filterOpt: Option[Query] = queryBuilder.buildFilterOpt(params, searchFilters, syndicationFilter) val withFilter = filterOpt.map { f => boolQuery() must (query) filter f diff --git a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala index 4bac2fb3f44..e6dd8fed5b0 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala @@ -5,7 +5,7 @@ import com.gu.mediaservice.lib.auth.{Authentication, Tier} import com.gu.mediaservice.lib.formatting.{parseDateFromQuery, printDateTime} import com.gu.mediaservice.model.usage.UsageStatus import com.gu.mediaservice.model.{Image, PrintUsageFilters, SyndicationStatus} -import lib.querysyntax.{Condition, Parser} +import lib.querysyntax.{AnyField, Condition, Match, Parser, Phrase, SimilarField, SimilarValue, Words} import org.joda.time.DateTime import play.api.libs.json.{Json, OWrites} import play.api.mvc.{AnyContent, Request} @@ -17,6 +17,67 @@ case class SearchResults(hits: Seq[(String, SourceWrapper[Image])], total: Long, case class AggregateSearchResults(results: Seq[BucketResult], total: Long) +case class AiQueryParts( + semanticQuery: Option[String], + filterConditions: List[Condition], + similarImageId: Option[String] +) { + def hasSimilarAndSemanticText: Boolean = similarImageId.nonEmpty && semanticQuery.nonEmpty + + // A KNN bit is the only thing AI search can rank by, so a query must contain either + // a text query (text KNN) or a similar image (image KNN) to be valid. + def hasKnn: Boolean = similarImageId.nonEmpty || semanticQuery.nonEmpty + + // Returns Left(error message) if the AI query cannot be satisfied, otherwise Right(()). + def validationError: Either[String, Unit] = { + if (hasSimilarAndSemanticText) + Left( + "AI search can't combine a similar image search with a text query because the two rankings can't be merged. " + + "Please use either a text query or a similar image, not both." + ) + else if (!hasKnn) + Left( + "AI search needs something to rank by. " + + "Add a text query or a similar image alongside your filters." + ) + else + Right(()) + } +} + +object AiQueryParts { + def from(conditions: List[Condition]): AiQueryParts = { + val initial = AiQueryParts( + semanticQuery = None, + filterConditions = List.empty, + similarImageId = None, + ) + + conditions.foldLeft(initial) { + case (parts, Match(AnyField, Words(value))) => + appendSemanticText(parts, value) + case (parts, Match(AnyField, Phrase(value))) => + appendSemanticText(parts, value) + case (parts, Match(SimilarField, SimilarValue(imageId))) if imageId.trim.nonEmpty => + parts.copy(similarImageId = Some(imageId.trim)) + case (parts, Match(SimilarField, _)) => + parts + case (parts, condition) => + parts.copy(filterConditions = parts.filterConditions :+ condition) + } + } + + private def appendSemanticText(parts: AiQueryParts, value: String): AiQueryParts = { + val trimmedValue = value.trim + + if (trimmedValue.isEmpty) parts + else { + val semanticQuery = parts.semanticQuery.fold(trimmedValue)(existing => s"$existing $trimmedValue") + parts.copy(semanticQuery = Some(semanticQuery)) + } + } +} + case class CompletionSuggestionResult(key: String, score: Float) object CompletionSuggestionResult { @@ -87,7 +148,9 @@ case class SearchParams( shouldFlagGraphicImages: Boolean = false, useAISearch: Option[Boolean] = None, vecWeight: Option[Double] = None -) +) { + lazy val aiQueryParts: AiQueryParts = AiQueryParts.from(structuredQuery) +} case class InvalidUriParams(message: String) extends Throwable object InvalidUriParams { diff --git a/media-api/app/lib/elasticsearch/QueryBuilder.scala b/media-api/app/lib/elasticsearch/QueryBuilder.scala index 7b53c02c4c2..ed8989282b1 100644 --- a/media-api/app/lib/elasticsearch/QueryBuilder.scala +++ b/media-api/app/lib/elasticsearch/QueryBuilder.scala @@ -1,7 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.lib.ImageFields -import com.gu.mediaservice.lib.elasticsearch.IndexSettings +import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.formatting.printDateTime import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.Agency @@ -12,6 +12,8 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.queries.matches.{MultiMatchQuery, MultiMatchQueryBuilderType} import lib.querysyntax._ import lib.MediaApiConfig +import scalaz.NonEmptyList +import scalaz.syntax.std.list._ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agency], config: MediaApiConfig) extends ImageFields with GridLogging { @@ -82,6 +84,9 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc matchNoneQuery() } } + case SimilarField => + logger.info(s"Cannot perform SIMILAR query on ${condition.value} outside AI search mode") + matchNoneQuery() } def makeQuery(conditions: List[Condition]) = conditions match { @@ -122,4 +127,87 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc listOfNestedToQueries(negationNested).foldLeft(queryWithNestedAndNormal) { case (q, negNestedQ) => q.withNot(negNestedQ) } } } + + def buildFilterOpt(params: SearchParams, searchFilters: SearchFilters, syndicationFilter: SyndicationFilter): Option[Query] = { + + val uploadTimeFilter = filters.date("uploadTime", params.since, params.until) + val lastModTimeFilter = filters.date("lastModified", params.modifiedSince, params.modifiedUntil) + val takenTimeFilter = filters.date("metadata.dateTaken", params.takenSince, params.takenUntil) + // we only inject filters if there are actual date parameters + val dateFilterList = List(uploadTimeFilter, lastModTimeFilter, takenTimeFilter).flatten.toNel + val dateFilter = dateFilterList.map(dateFilters => filters.and(dateFilters.list.toList: _*)) + + val idsFilter = params.ids.map(filters.ids) + val labelFilter = params.labels.toNel.map(filters.terms("labels", _)) + val metadataFilter = params.hasMetadata.map(metadataField).toNel.map(filters.exists) + val archivedFilter = params.archived.map(filters.existsOrMissing(editsField("archived"), _)) + val hasExports = params.hasExports.map(filters.existsOrMissing("exports", _)) + val hasIdentifier = params.hasIdentifier.map(idName => filters.exists(NonEmptyList(identifierField(idName)))) + val missingIdentifier = params.missingIdentifier.map(idName => filters.missing(NonEmptyList(identifierField(idName)))) + val uploadedByFilter = params.uploadedBy.map(uploadedBy => filters.terms("uploadedBy", NonEmptyList(uploadedBy))) + val simpleCostFilter = params.free.flatMap(free => if (free) searchFilters.freeFilter else searchFilters.nonFreeFilter) + val costFilter = params.payType match { + case Some(PayType.Free) => searchFilters.freeFilter + case Some(PayType.MaybeFree) => searchFilters.maybeFreeFilter + case Some(PayType.Pay) => searchFilters.nonFreeFilter + case _ => None + } + + val printUsageFilter = params.printUsageFilters.map(searchFilters.printUsageFilters) + + val hasRightsCategory = params.hasRightsCategory.filter(_ == true).map(_ => searchFilters.hasRightsCategoryFilter) + + val validityFilter = params.valid.map(valid => if (valid) searchFilters.validFilter else searchFilters.invalidFilter) + + val persistFilter = params.persisted map { + case true => searchFilters.persistedFilter + case false => searchFilters.nonPersistedFilter + } + + val usageFilter: Iterable[Query] = + params.usageStatus.toNel.map(status => filters.terms("usagesStatus", status.map(_.toString))).toOption ++ + params.usagePlatform.toNel.map(filters.terms("usagesPlatform", _)).toOption + + val syndicationStatusFilter = params.syndicationStatus.map(status => syndicationFilter.statusFilter(status)) + + // Port of special case code in elastic1 sorts. Using the dateAddedToCollection sort implies an additional filter for reasons unknown + val dateAddedToCollectionFilter = { + params.orderBy match { + case Some("dateAddedToCollection") => { + val pathHierarchyOpt = params.structuredQuery.flatMap { + case Match(HierarchyField, Phrase(value)) => Some(value) + case _ => None + }.headOption + + pathHierarchyOpt.map { pathHierarchy => + termQuery("collections.pathHierarchy", pathHierarchy) + } + } + case _ => None + } + } + + val filterOpt = ( + metadataFilter.toOption.toList + ++ persistFilter + ++ labelFilter.toOption + ++ archivedFilter + ++ uploadedByFilter + ++ idsFilter + ++ validityFilter + ++ simpleCostFilter + ++ costFilter + ++ hasExports + ++ hasIdentifier + ++ missingIdentifier + ++ dateFilter.toOption + ++ usageFilter + ++ hasRightsCategory + ++ searchFilters.tierFilter(params.tier) + ++ syndicationStatusFilter + ++ dateAddedToCollectionFilter + ++ printUsageFilter + ).toNel.map(filter => filter.list.toList.reduceLeft(filters.and(_, _))).toOption + filterOpt + } } diff --git a/media-api/app/lib/querysyntax/QuerySyntax.scala b/media-api/app/lib/querysyntax/QuerySyntax.scala index cb62b5346e6..73dfc89080d 100644 --- a/media-api/app/lib/querysyntax/QuerySyntax.scala +++ b/media-api/app/lib/querysyntax/QuerySyntax.scala @@ -33,6 +33,7 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { def Filter = rule { HasMatch ~> Match | IsMatch ~> Match | + SimilarMatch ~> Match | DateConstraintMatch | DateRangeMatch ~> Match | AtMatch | FileTypeMatch ~> Match | @@ -51,6 +52,11 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { def IsFieldName = rule { "is" } def IsMatchValue = rule { QuotedString ~> IsValue | String ~> IsValue } + def SimilarMatch = rule { SimilarMatchField ~ ':' ~ SimilarMatchValue } + def SimilarMatchField = rule { capture(SimilarFieldName) ~> (_ => SimilarField) } + def SimilarFieldName = rule { "similar" } + def SimilarMatchValue = rule { QuotedString ~> SimilarValue | String ~> SimilarValue } + def NestedMatch = rule { ParentField ~ "@" ~ NestedField ~ ':' ~ ExactMatchValue } def NestedDateMatch = rule { ParentField ~ "@" ~ DateConstraintMatch ~> ( (parentField: Field, dateMatch: Match) => { diff --git a/media-api/app/lib/querysyntax/model.scala b/media-api/app/lib/querysyntax/model.scala index fe22886154d..871cb79e5ed 100644 --- a/media-api/app/lib/querysyntax/model.scala +++ b/media-api/app/lib/querysyntax/model.scala @@ -15,6 +15,7 @@ final case class SingleField(name: String) extends Field final case class MultipleField(names: List[String]) extends Field final case object HasField extends Field final case object IsField extends Field +final case object SimilarField extends Field sealed trait Value final case class Words(string: String) extends Value @@ -23,3 +24,4 @@ final case class Date(date: DateTime) extends Value final case class DateRange(startDate: DateTime, endDate: DateTime) extends Value final case class HasValue(string: String) extends Value final case class IsValue(string: String) extends Value +final case class SimilarValue(string: String) extends Value diff --git a/media-api/conf/routes b/media-api/conf/routes index d868fd4c400..2b22b7b6692 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -39,6 +39,9 @@ GET /usage/suppliers/:id controllers. GET /usage/quotas controllers.UsageController.quotas GET /usage/quotas/:id controllers.UsageController.quotaForImage(id: String) +# configuration +GET /configuration/crop-variations controllers.ConfigurationController.cropVariations + # Management GET /management/healthcheck com.gu.mediaservice.lib.management.ElasticSearchHealthCheck.healthCheck GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest diff --git a/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala b/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala new file mode 100644 index 00000000000..306b670fd68 --- /dev/null +++ b/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala @@ -0,0 +1,130 @@ +package lib.elasticsearch + +import com.gu.mediaservice.lib.ImageFields +import lib.querysyntax.{AnyField, IsField, IsValue, Match, Negation, NegationNested, Nested, Parser, Phrase, SingleField, Words} +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class AiQueryPartsTest extends AnyFunSpec with Matchers with ImageFields { + + // The parser always appends these two negations to every parsed query, so they + // always show up as part of the derived filter conditions. + private val standardNegations = List( + Negation(Match(IsField, IsValue("deleted"))), + NegationNested(Nested(SingleField("usages"), SingleField("usages.status"), Phrase("replaced"))) + ) + + private val jpegFilter = Match(SingleField(getFieldPath("mimeType")), Words("image/jpeg")) + private val negatedAngela = Negation(Match(AnyField, Words("angela"))) + + // Build the AiQueryParts the same way the controller does: parse the raw query + // string with the production parser, then derive the AI search parts from it. + private def aiPartsFor(query: String): AiQueryParts = + AiQueryParts.from(Parser.run(query)) + + describe("AiQueryParts validation") { + + describe("valid queries (each contains a KNN ranking signal)") { + + it("allows a similar-image-only query (image KNN)") { + val parts = aiPartsFor("similar:abc123") + + parts.similarImageId should be(Some("abc123")) + parts.semanticQuery should be(None) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + + it("allows a text-only query (text KNN)") { + val parts = aiPartsFor("keir starmer") + + parts.semanticQuery should be(Some("keir starmer")) + parts.similarImageId should be(None) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + + it("allows chips/filters combined with a text query (pre-filter then text KNN)") { + val parts = aiPartsFor("fileType:jpeg keir starmer") + + parts.semanticQuery should be(Some("keir starmer")) + parts.filterConditions should be(jpegFilter :: standardNegations) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + + it("allows chips/filters combined with a similar image (pre-filter then image KNN)") { + val parts = aiPartsFor("similar:abc123 fileType:jpeg") + + parts.similarImageId should be(Some("abc123")) + parts.filterConditions should be(jpegFilter :: standardNegations) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + + it("allows a negative term combined with a text query (negative is a lexical pre-filter, then text KNN)") { + val parts = aiPartsFor("keir starmer -angela") + + parts.semanticQuery should be(Some("keir starmer")) + // The negation is treated as a filter condition, not a KNN ranking signal. + parts.filterConditions should be(negatedAngela :: standardNegations) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + + it("allows a negative term combined with a similar image (negative pre-filter, then image KNN)") { + val parts = aiPartsFor("similar:abc123 -angela") + + parts.similarImageId should be(Some("abc123")) + parts.filterConditions should be(negatedAngela :: standardNegations) + parts.hasKnn should be(true) + parts.validationError should be(Right(())) + } + } + + describe("invalid queries that combine two rankings") { + + it("rejects a similar image combined with a text query (rankings can't be merged)") { + val parts = aiPartsFor("similar:abc123 keir starmer") + + parts.similarImageId should be(Some("abc123")) + parts.semanticQuery should be(Some("keir starmer")) + parts.hasSimilarAndSemanticText should be(true) + parts.validationError.isLeft should be(true) + } + } + + describe("invalid queries that contain no KNN ranking signal") { + + it("rejects a chips/filters-only query") { + val parts = aiPartsFor("fileType:jpeg") + + parts.semanticQuery should be(None) + parts.similarImageId should be(None) + parts.filterConditions should be(jpegFilter :: standardNegations) + parts.hasKnn should be(false) + parts.validationError.isLeft should be(true) + } + + it("rejects a negative-only query") { + val parts = aiPartsFor("-angela") + + parts.semanticQuery should be(None) + parts.similarImageId should be(None) + parts.filterConditions should be(negatedAngela :: standardNegations) + parts.hasKnn should be(false) + parts.validationError.isLeft should be(true) + } + + it("rejects a negative term combined with chips/filters") { + val parts = aiPartsFor("-angela fileType:jpeg") + + parts.semanticQuery should be(None) + parts.similarImageId should be(None) + parts.filterConditions should be(negatedAngela :: jpegFilter :: standardNegations) + parts.hasKnn should be(false) + parts.validationError.isLeft should be(true) + } + } + } +} diff --git a/media-api/test/lib/querysyntax/ParserTest.scala b/media-api/test/lib/querysyntax/ParserTest.scala index 4e5aa9ddbfa..4cfea97da10 100644 --- a/media-api/test/lib/querysyntax/ParserTest.scala +++ b/media-api/test/lib/querysyntax/ParserTest.scala @@ -486,6 +486,24 @@ class ParserTest extends AnyFunSpec with Matchers with BeforeAndAfter with Image } } + describe("similar filter") { + it("should parse similar queries") { + Parser.run("similar:abc123") should be (List( + Match(SimilarField, SimilarValue("abc123")), + ) ++ standardNegations + ) + } + + it("should parse similar queries with other filters and text") { + Parser.run("similar:abc123 fileType:jpeg hello") should be (List( + Match(SimilarField, SimilarValue("abc123")), + Match(SingleField(getFieldPath("mimeType")), Words("image/jpeg")), + Match(AnyField, Words("hello")), + ) ++ standardNegations + ) + } + } + describe("fileType filter") { it("should find jpegs images") { Parser.run("fileType:jpeg") should be (List( diff --git a/scripts/inadvisable-metadata-backup/.ExifTool_config b/scripts/inadvisable-metadata-backup/.ExifTool_config deleted file mode 100644 index a87b95a0905..00000000000 --- a/scripts/inadvisable-metadata-backup/.ExifTool_config +++ /dev/null @@ -1,18 +0,0 @@ -# A config file for exiftool, based on the example and work by @paperboyo -# It's perl -%Image::ExifTool::UserDefined = ( - 'Image::ExifTool::XMP::Main' => { - gridImage => { # <-- must be the same as the NAMESPACE prefix - SubDirectory => { - TagTable => 'Image::ExifTool::UserDefined::gridImage', - }, - }, - }, -); -%Image::ExifTool::UserDefined::gridImage = ( - GROUPS => { 0 => 'XMP', 1 => 'XMP-gridImage', 2 => 'Image' }, - NAMESPACE => { 'gridImage' => 'com.guardian.xmp.gridImage' }, - WRITABLE => 'string', - FileMetadata => { Writable => 'string' }, -); -1; #end diff --git a/scripts/inadvisable-metadata-backup/ImageMetadata.ts b/scripts/inadvisable-metadata-backup/ImageMetadata.ts deleted file mode 100644 index 54b370328bc..00000000000 --- a/scripts/inadvisable-metadata-backup/ImageMetadata.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { WriteTags } from "exiftool-vendored"; - -export interface ImageMetadata { - dateTaken?: string; - description?: string; - credit?: string; - byline?: string; - bylineTitle?: string; - title?: string; - copyright?: string; - suppliersReference?: string; - source?: string; - specialInstructions?: string; - keywords: string[]; - subLocation?: string; - city?: string; - state?: string; - country?: string; - subjects: string[]; - peopleInImage: string[]; -} - -const impartial = (x: object): any => - Object.fromEntries(Object.entries(x).filter(([k]) => k !== undefined)); - -export const toWriteTags = ( - meta: ImageMetadata, - fileMetadata: unknown -): WriteTags => { - const legal: Partial = { - DateTimeOriginal: meta.dateTaken, - Source: meta.source, - "Sub-location": meta.subLocation, - City: meta.city, - Country: meta.country, - Copyright: meta.copyright - }; - const specificAndLimited = { - SupplementalCategories: meta.subjects, - "IPTC:Credit":meta.credit, - "XMP-iptcExt:PersonInImage": meta.peopleInImage, - "XMP-gridImage:FileMetadata": JSON.stringify(fileMetadata), - "XMP-photoshop:Credit": meta.credit, - "XMP-photoshop:Instructions": meta.specialInstructions, - "XMP-photoshop:Source": meta.source, - "XMP-dc:subject": meta.keywords, - "XMP-dc:description": meta.description, - "XMP-dc:creator": meta.byline, - "XMP-photoshop:AuthorsPosition": meta.bylineTitle, - "XMP-photoshop:Headline": meta.title, - "XMP-dc:Rights": meta.copyright, - "XMP-photoshop:TransmissionReference": meta.suppliersReference, - }; - return impartial({ ...legal, ...specificAndLimited }); -}; diff --git a/scripts/inadvisable-metadata-backup/README.md b/scripts/inadvisable-metadata-backup/README.md deleted file mode 100644 index 6bf2ecb3bb0..00000000000 --- a/scripts/inadvisable-metadata-backup/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Inadvidable Metadata Backup Tool - -Suppose you found that a subset of images have been modified since upload and no longer have their exif/iptc/xmp metadata. - -This is bad! - -## Do not fear - -We have the technology, we have the pertitnent metadata in elasticsearch. - -This script fetches an image, backs it up to it's path + `_backup` in s3, and then fetches it's metadata from the grid using grid-cli. - -It then uses exiftool to put pertinent metadata back in the image for reingestion, and the remainder of the data in a FileMetadata XMP tag. - -## Running - -Install the dependencies by running yarn. - -Ensure [grid-cli](https://github.com/guardian/grid-cli) is installed and configured. - -`AWS_PROFILE=$aws_profile ./run.sh $image_bucket $id` - -Where `$aws_profile` is the AWS profile your bucket is in, `$image_bucket` is your image bucket and `$id` is the image. - -### Extracting Metadata - -`grid-cli image:download -d=. $id` -`yarn extract $id` diff --git a/scripts/inadvisable-metadata-backup/extract.ts b/scripts/inadvisable-metadata-backup/extract.ts deleted file mode 100644 index b8c86d9ef02..00000000000 --- a/scripts/inadvisable-metadata-backup/extract.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { exiftool } from "exiftool-vendored"; -import path from "path"; - -const [, , fileName] = process.argv; - -(async () => { - const t = await exiftool.read(path.resolve(fileName)) as any - console.log(t.FileMetadata) - await exiftool.end(); -})(); diff --git a/scripts/inadvisable-metadata-backup/package.json b/scripts/inadvisable-metadata-backup/package.json deleted file mode 100644 index dd544e144b5..00000000000 --- a/scripts/inadvisable-metadata-backup/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "exifthing", - "version": "1.0.0", - "main": "index.js", - "license": "MIT", - "dependencies": { - "@types/node": "^14.14.37", - "aws-sdk": "^2.876.0", - "exiftool-vendored": "^22.2.0", - "shell-escape": "^0.2.0", - "ts-node": "^9.1.1", - "typescript": "^4.2.3" - }, - "scripts": { - "update":"ts-node update.ts", - "s3":"ts-node s3.ts" - } -} diff --git a/scripts/inadvisable-metadata-backup/run.sh b/scripts/inadvisable-metadata-backup/run.sh deleted file mode 100755 index 98a7a88f547..00000000000 --- a/scripts/inadvisable-metadata-backup/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e -x - -ID=$2 -BUCKET=$1 - -echo "Processing $ID in $BUCKET" - -grid image:download -d=. $ID -grid image:get --hydrate $ID > $ID.json - -exiftool -all= $ID -yarn update $ID -yarn s3 $BUCKET $ID - -echo "DONE" diff --git a/scripts/inadvisable-metadata-backup/s3.ts b/scripts/inadvisable-metadata-backup/s3.ts deleted file mode 100644 index d993a31adc2..00000000000 --- a/scripts/inadvisable-metadata-backup/s3.ts +++ /dev/null @@ -1,32 +0,0 @@ -process.env.AWS_PROFILE = "media-service" -import { S3 } from "aws-sdk"; -import { promises as fs } from "fs"; -import path from "path"; - -let client = new S3({ region: "eu-west-1" }); -const [, , Bucket, id] = process.argv; - -const Key = `${id.substring(0, 6).split("").join("/")}/${id}`; - -(async () => { - console.log(`Attempting to backup and replace ${Bucket} object ${Key}`) - await client - .copyObject({ - Bucket, - CopySource: `${Bucket}/${Key}`, - Key: `${Key}_backup`, - }) - .promise(); - console.log(`Backed up ${Key} to ${Key}_backup`); - const { Metadata } = await client.headObject({ Bucket, Key }).promise(); - - await client - .upload({ - Bucket, - Key: `${Key}`, - Metadata, - Body: await fs.readFile(path.resolve("./" + id)), - }) - .promise(); - console.log(`Uploaded ${id} to ${Key}`); -})(); diff --git a/scripts/inadvisable-metadata-backup/tsconfig.json b/scripts/inadvisable-metadata-backup/tsconfig.json deleted file mode 100644 index f92d2ba967e..00000000000 --- a/scripts/inadvisable-metadata-backup/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true - } -} diff --git a/scripts/inadvisable-metadata-backup/update.ts b/scripts/inadvisable-metadata-backup/update.ts deleted file mode 100644 index b591d2e3aa7..00000000000 --- a/scripts/inadvisable-metadata-backup/update.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { exiftool } from "exiftool-vendored"; -import { toWriteTags } from "./ImageMetadata"; -import path from "path"; - -const [, , id] = process.argv; - -console.log( - `Ran with ${id}, expecting ${id}.json and ${id} to be in working directory.` -); - -const { data } = require(path.resolve(`${id}.json`)); -const { fileMetadata, originalMetadata } = data; - -(async () => { - const tags = toWriteTags(originalMetadata, fileMetadata) - console.log(JSON.stringify(tags,null,2)) - await exiftool.write(`${id}`, tags); - await exiftool.end(); - console.log("Updated local metadata."); -})(); diff --git a/scripts/inadvisable-metadata-backup/yarn.lock b/scripts/inadvisable-metadata-backup/yarn.lock deleted file mode 100644 index 1e87c606e6c..00000000000 --- a/scripts/inadvisable-metadata-backup/yarn.lock +++ /dev/null @@ -1,222 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@photostructure/tz-lookup@^8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@photostructure/tz-lookup/-/tz-lookup-8.0.0.tgz#9742eedc61e6cf75539f5adbb3d0d90dab7294ee" - integrity sha512-D5ggPEWSNGEKzKTx6+Gn0NZXHQ4ywgRd2p2h7i/tjEmkv/uJ9SzQd0o7v7FzEAt4bP3dxDoWm43aPfUs9qMOGg== - -"@types/luxon@^3.3.2": - version "3.3.2" - resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-3.3.2.tgz#f6e3524c2486b949a4db445e85d93c8e9886dfe2" - integrity sha512-l5cpE57br4BIjK+9BSkFBOsWtwv6J9bJpC7gdXIzZyI0vuKvNTk0wZZrkQxMGsUAuGW9+WMNWF2IJMD7br2yeQ== - -"@types/node@^14.14.37": - version "14.14.37" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.37.tgz#a3dd8da4eb84a996c36e331df98d82abd76b516e" - integrity sha512-XYmBiy+ohOR4Lh5jE379fV2IU+6Jn4g5qASinhitfyO71b/sCo6MKsMLF5tc7Zf2CE8hViVQyYSobJNke8OvUw== - -arg@^4.1.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" - integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== - -aws-sdk@^2.876.0: - version "2.877.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.877.0.tgz#e580c08aeba8cb148d1dc0ee16348a410bf6852e" - integrity sha512-cNp526SUmJ0WZs40R4Zyk19FYWyVpyidJHOLf+6a+Uv0zG0PvH5257iAWMp81Qz3Sjb11y+wMSOwzCIp7SAZig== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - -base64-js@^1.0.2: - version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - -batch-cluster@^12.1.0: - version "12.1.0" - resolved "https://registry.yarnpkg.com/batch-cluster/-/batch-cluster-12.1.0.tgz#697f3e11d7bc49d6befea139025431efe0e1e16f" - integrity sha512-whGyJU4tr7kyg2USByu0/51mML5HsLAeNz5s03kMDYZNsQsGgDJgI47RdY3r7MciCjPkTaTD5O4eOVqOfEO7pg== - -buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== - -buffer@4.9.2: - version "4.9.2" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" - integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - isarray "^1.0.0" - -create-require@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" - integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== - -diff@^4.0.1: - version "4.0.4" - resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.4.tgz#7a6dbfda325f25f07517e9b518f897c08332e07d" - integrity sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ== - -events@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= - -exiftool-vendored.exe@12.65.0: - version "12.65.0" - resolved "https://registry.yarnpkg.com/exiftool-vendored.exe/-/exiftool-vendored.exe-12.65.0.tgz#28898bad74228be90f622068dda0d089af581aee" - integrity sha512-VDTSW3/u5bdLlg516g1oTypq2Sxd3I2pWTzdd5EmDtSjmvvBCLyDlMpv4Gnz8dnlQTRsEqwIgv/TAtdWykwEBg== - -exiftool-vendored.pl@12.65.0: - version "12.65.0" - resolved "https://registry.yarnpkg.com/exiftool-vendored.pl/-/exiftool-vendored.pl-12.65.0.tgz#ad4f19da9be56055e5fb9dd40b395ccd998ff754" - integrity sha512-BpR+rwKLWqUAPbsW17fw+8FAmyijkMhjaLu3fWU2QX6rpBJnOfn+lQp4Txkq44avL1LDV+eQ8pbWXyimjkPw0Q== - -exiftool-vendored@^22.2.0: - version "22.2.0" - resolved "https://registry.yarnpkg.com/exiftool-vendored/-/exiftool-vendored-22.2.0.tgz#af2065d141de8d7edffce5a3b1efce0d06a27d85" - integrity sha512-NOGwIuQ386VQSjrV4YVZ0QI1dmoDVESfKmXsQwWBtcP0IBCrC9LJPhw5p3jR3jY9ehYi8sEdYWL1kI6fb/Ojaw== - dependencies: - "@photostructure/tz-lookup" "^8.0.0" - "@types/luxon" "^3.3.2" - batch-cluster "^12.1.0" - he "^1.2.0" - luxon "^3.4.2" - optionalDependencies: - exiftool-vendored.exe "12.65.0" - exiftool-vendored.pl "12.65.0" - -he@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" - integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== - -ieee754@1.1.13: - version "1.1.13" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" - integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== - -ieee754@^1.1.4: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" - integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== - -isarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= - -luxon@^3.4.2: - version "3.4.3" - resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.3.tgz#8ddf0358a9492267ffec6a13675fbaab5551315d" - integrity sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg== - -make-error@^1.1.1: - version "1.3.6" - resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" - integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== - -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= - -sax@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= - -sax@>=0.6.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== - -shell-escape@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133" - integrity sha1-aP0CXrBJC09WegJ/C/IkgLX4QTM= - -source-map-support@^0.5.17: - version "0.5.19" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - -source-map@^0.6.0: - version "0.6.1" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" - integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== - -ts-node@^9.1.1: - version "9.1.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d" - integrity sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg== - dependencies: - arg "^4.1.0" - create-require "^1.1.0" - diff "^4.0.1" - make-error "^1.1.1" - source-map-support "^0.5.17" - yn "3.1.1" - -typescript@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.2.3.tgz#39062d8019912d43726298f09493d598048c1ce3" - integrity sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw== - -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= - dependencies: - punycode "1.3.2" - querystring "0.2.0" - -uuid@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= - -yn@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" - integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/EsScript.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/EsScript.scala index 57d1d4ee26d..26f7d54da1a 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/EsScript.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/EsScript.scala @@ -197,15 +197,19 @@ object Reindex extends EsScript { object UpdateMapping extends EsScript { def run(esUrl: String, extraArgs: List[String]): Unit = { + val includeDenseVectors = !extraArgs.contains("--no-dense-vectors") + val remainingArgs = extraArgs.filterNot(_ == "--no-dense-vectors") - object MappingsClient extends EsClient(esUrl) { + println(s"includeDenseVectorMappings: $includeDenseVectors") + + object MappingsClient extends EsClient(esUrl, includeDenseVectors) { def updateMappings(specifiedIndex: Option[String]): Unit = { val index = specifiedIndex.getOrElse(imagesCurrentAlias) println(s"Updating mapping on index: $index") val result = client.execute { putMapping(Indexes(index)) as { - Mappings.imageMapping.properties + Mappings.imageMapping(includeDenseVectors).properties } }.await @@ -219,11 +223,11 @@ object UpdateMapping extends EsScript { } - MappingsClient.updateMappings(extraArgs.headOption) + MappingsClient.updateMappings(remainingArgs.headOption) } def usageError: Nothing = { - System.err.println("Usage: UpdateMapping ") + System.err.println("Usage: UpdateMapping [] [--no-dense-vectors]") sys.exit(1) } } @@ -422,7 +426,7 @@ abstract class EsScript { run(esUrl, extraArgs) } - class EsClient(val url: String) extends ElasticSearchClient { + class EsClient(val url: String, val includeDenseVectorMappings: Boolean = true) extends ElasticSearchClient { override def imagesCurrentAlias = esImagesAlias override def shards = esShards override def replicas = esReplicas diff --git a/thrall/app/lib/elasticsearch/ElasticSearch.scala b/thrall/app/lib/elasticsearch/ElasticSearch.scala index e7defb3437f..29f82489c84 100644 --- a/thrall/app/lib/elasticsearch/ElasticSearch.scala +++ b/thrall/app/lib/elasticsearch/ElasticSearch.scala @@ -41,6 +41,7 @@ class ElasticSearch( lazy val url: String = config.url lazy val shards: Int = config.shards lazy val replicas: Int = config.replicas + lazy val includeDenseVectorMappings: Boolean = config.includeDenseVectorMappings def migrationAwareUpdater[REQUEST, RESPONSE]( @@ -78,9 +79,13 @@ class ElasticSearch( executeAndLog(request, s"Setting migration info on image id: ${imageId}") } + private def prepareImageJson(json: JsObject): JsObject = + if (includeDenseVectorMappings) json + else json - "embedding" + def directInsert(image: Image, indexName: String)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[ElasticSearchInsertResponse] = executeAndLog( - indexInto(indexName).id(image.id).source(Json.stringify(Json.toJson(image))), + indexInto(indexName).id(image.id).source(Json.stringify(prepareImageJson(Json.toJson(image).as[JsObject]))), s"ES6 indexing image ${image.id} into index '$indexName'" ).map(indexResponse => ElasticSearchInsertResponse(indexResponse.result.index) @@ -92,14 +97,14 @@ class ElasticSearch( // On insert, we know we will not have a lastModified to consider, so we always take the one we get val insertImage = image.copy(lastModified = Some(lastModified)) - val insertImageAsJson = Json.toJson(insertImage) + val insertImageAsJson = prepareImageJson(Json.toJson(insertImage).as[JsObject]) def runUpsertIntoIndex(indexAlias: String, maybeEsInfo: Option[JsObject]) = { val esInfo = maybeEsInfo.getOrElse(JsObject.empty) // On update, we do not want to take the one we have been given unless it is newer - see updateLastModifiedScript script val updateImage = image.copy(lastModified = None) - val upsertImageAsJson = Json.toJson(updateImage) + val upsertImageAsJson = prepareImageJson(Json.toJson(updateImage).as[JsObject]) val painlessSource = // If there are old identifiers, then merge any new identifiers into old and use the merged results as the new identifiers @@ -214,18 +219,24 @@ class ElasticSearch( def updateEmbedding(id: String, embedding: Embedding, lastModified: DateTime) (implicit ex: ExecutionContext, logMarker: LogMarker): List[Future[ElasticSearchUpdateResponse]] = { - val replaceEmbeddingScript = "ctx._source.embedding = params.embedding;" + if (!includeDenseVectorMappings) { + logger.info(s"Skipping embedding update for image $id: dense vector mappings are disabled") + List(Future.successful(ElasticSearchUpdateResponse())) + } else { - val scriptSource = loadPainless(replaceEmbeddingScript) + val replaceEmbeddingScript = "ctx._source.embedding = params.embedding;" - val embeddingParameter = asNestedMap(Json.toJson(embedding)) + val scriptSource = loadPainless(replaceEmbeddingScript) - val eventualUpdateResponse = migrationAwareUpdater( - requestFromIndexName = indexName => prepareUpdateRequest(indexName, id, scriptSource, lastModified, ("embedding", embeddingParameter)), - logMessageFromIndexName = indexName => s"ES6 updating embedding on image $id in index $indexName" - ).incrementOnFailure(metrics.map(_.failedEmbeddingUpdates)) { case _ => true } + val embeddingParameter = asNestedMap(Json.toJson(embedding)) - List(eventualUpdateResponse.map(_ => ElasticSearchUpdateResponse())) + val eventualUpdateResponse = migrationAwareUpdater( + requestFromIndexName = indexName => prepareUpdateRequest(indexName, id, scriptSource, lastModified, ("embedding", embeddingParameter)), + logMessageFromIndexName = indexName => s"ES6 updating embedding on image $id in index $indexName" + ).incrementOnFailure(metrics.map(_.failedEmbeddingUpdates)) { case _ => true } + + List(eventualUpdateResponse.map(_ => ElasticSearchUpdateResponse())) + } } def applyImageMetadataOverride(id: String, metadata: Edits, lastModified: DateTime)