diff --git a/integration-tests/README.md b/integration-tests/README.md index f9e22ec..f0a5f3f 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -12,7 +12,7 @@ ## Dependencies -The integration tests download test data from a sepcific commit hash in the [bestbets-content GitHub repository](https://github.com/nciocpl/bestbets-content). To retrieve data from a different commit, edit `bin/load-integration-data.sh` and modify the value of the `BESTBETS_CONTENT_COMMIT` environment variable. +The integration tests download test data from a specific commit hash in the [bestbets-content GitHub repository](https://github.com/nciocpl/bestbets-content). To retrieve data from a different commit, edit `bin/load-integration-data.sh` and modify the value of the `BESTBETS_CONTENT_COMMIT` environment variable. the integration data is loaded by referencing the [bestbets-loader](https://github.com/nciocpl/bestbets-loader) as a git submodule. It is downloaded, installed, and executed via `load-integration-data.sh`. To update the loader's version, go to the `bin/bestbets-loader` directory, fetch and checkout the newer version, and commit the change. (See also: [the submodule reference in the git manual](https://git-scm.com/book/en/v2/Git-Tools-Submodules).) @@ -20,5 +20,5 @@ the integration data is loaded by referencing the [bestbets-loader](https://gith * [Docs for understanding how to run Karate standalone](https://github.com/intuit/karate/blob/6de466bdcf105d72450a40cf31b8adb5c043037d/karate-netty/README.md#standalone-jar) * Specifically this has to do with the magic naming of the logging config which is really why I am posting this here! * We have docker for dev testing because ES will no longer run on higher Java versions, this is the easiest way to get it up and running. -* .NET running locally on a Mac cannot talk to ES because of how NEST always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac. +* .NET running locally on a Mac cannot talk to ES because of how the client always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac. * You need to use the `--force-recreate` option to `docker-compose up` or run `docker-compose rm` after shutting down the cluster. If the elasticsearch container is not removed, it keeps its data, and any restarts will leave the cluster in a bad state. diff --git a/integration-tests/docker-bestbets-api/api/Dockerfile b/integration-tests/docker-bestbets-api/api/Dockerfile index ddb14d0..1372040 100644 --- a/integration-tests/docker-bestbets-api/api/Dockerfile +++ b/integration-tests/docker-bestbets-api/api/Dockerfile @@ -4,8 +4,8 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 # Get the NIH Root certificates and install them so we work on VPN # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx RUN mkdir /usr/local/share/ca-certificates/nih \ - && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \ - && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \ + && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-CA-1A%281%29.crt \ + && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-ROOT-1A.cer \ && update-ca-certificates WORKDIR /app diff --git a/integration-tests/docker-bestbets-api/docker-compose.yml b/integration-tests/docker-bestbets-api/docker-compose.yml index 864ceaa..5046472 100644 --- a/integration-tests/docker-bestbets-api/docker-compose.yml +++ b/integration-tests/docker-bestbets-api/docker-compose.yml @@ -9,8 +9,11 @@ services: environment: - discovery.type=single-node - ES_JAVA_OPTS=-Xms750m -Xmx750m - # Turn off warnings about not using HTTPS. + # Turn security settings off for testing. (Allows http instead of https.) - xpack.security.enabled=false + - xpack.security.transport.ssl.enabled=false + - xpack.security.http.ssl.enabled=false + - http.host=0.0.0.0 ## These exposed ports are for debugging only. .NET + ## Docker + MacOS == bad scene. (.NET always wants to ## use the hosts name, and on a mac that is actually @@ -41,6 +44,7 @@ services: - ../../integration-tests/docker-bestbets-api/api/runtime/hosting.json:/app/src/NCI.OCPL.Api.BestBets/hosting.json # Use the user's existing GitHub credentials - ~/.nuget/NuGet/NuGet.Config:/root/.nuget/NuGet/NuGet.Config + ## NOTE: This does NOT mean that this machine will wait ## for elasticsearch to be running, just that the ## elasticsearch container should be running first. diff --git a/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile b/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile index 29e6fcc..5866f2b 100644 --- a/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile +++ b/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile @@ -1,9 +1,12 @@ -FROM elasticsearch:7.17.28 +FROM elasticsearch:8.19.12 + +USER root # Get the NIH Root certificates and install them so we work on VPN # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx RUN mkdir /usr/local/share/ca-certificates/nih \ - && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \ - && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \ + && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-CA-1A%281%29.crt \ + && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-ROOT-1A.cer \ && update-ca-certificates +USER elasticsearch diff --git a/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs b/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs index 838420e..f2674db 100644 --- a/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs +++ b/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs @@ -8,9 +8,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using Nest; -using Elasticsearch.Net; - using NCI.OCPL.Api.Common; namespace NCI.OCPL.Api.BestBets.Controllers @@ -91,7 +88,7 @@ public async Task Get(string collection, string language, str if (String.IsNullOrWhiteSpace(term)) throw new APIErrorException(400, "You must supply a search term"); - // Term comes from from a catch-all parameter, so make sure it's been decoded. + // Term comes from a catch-all parameter, so make sure it's been decoded. term = WebUtility.UrlDecode(term); // Step 1. Remove Punctuation diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs index 1104408..703c671 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs @@ -1,4 +1,3 @@ - using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsMatchService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsMatchService.cs index 21a1c1e..6567a5f 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsMatchService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsMatchService.cs @@ -8,11 +8,11 @@ namespace NCI.OCPL.Api.BestBets public interface IBestBetsMatchService { /// - /// Gets a list of the BestBet Category IDs that matched our term - /// + /// Gets a list of the BestBet Category IDs that matched our term + /// /// The collection to use. This will be 'live' or 'preview'. /// The two-character language code to constrain the matches to - /// A term that have been cleaned of punctuation and special characters + /// A term that has been cleaned of punctuation and special characters /// An array of category ids Task GetMatches(string collection, string language, string cleanedTerm); } diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IESSearchOptions.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IESSearchOptions.cs index 96d1659..7ee0c5b 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IESSearchOptions.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IESSearchOptions.cs @@ -17,7 +17,7 @@ public interface IESSearchOptions string Username {get; set;} /// - /// Gets and sets the Username for authenticating to the ES server + /// Gets and sets the Password for authenticating to the ES server /// string Password {get; set;} } diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs index 1484076..2aadada 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs @@ -1,5 +1,4 @@ - -using System.Threading.Tasks; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets { diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs index f4da30e..67bef13 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets { diff --git a/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs b/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs index e711d89..0f8f511 100644 --- a/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs +++ b/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs @@ -1,36 +1,34 @@ -using System; -using Nest; +using System.Text.Json.Serialization; namespace NCI.OCPL.Api.BestBets { /// /// Represents Display information about a Best Bet /// - [ElasticsearchType(RelationName = "categorydisplay")] public class BestBetsCategoryDisplay : IBestBetDisplay { /// /// Gets or sets the name of the category for this Best Bet Match /// - [Text(Name = "name")] + [JsonPropertyName("name")] public string Name { get; set; } /// /// Gets or sets the content ID of the category of this match /// - [Text(Name = "contentid")] + [JsonPropertyName("contentid")] public string ID { get; set; } /// /// Gets or sets the HTML for display for this category /// - [Text(Name = "content")] + [JsonPropertyName("content")] public string HTML { get; set; } /// /// Gets the weight of this category to determine ordering on display /// - [Text(Name = "weight")] + [JsonPropertyName("weight")] public int Weight { get; set; } /// diff --git a/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs b/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs index 6b157ae..22d960f 100644 --- a/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs +++ b/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs @@ -1,56 +1,53 @@ -using System; - -using Nest; +using System.Text.Json.Serialization; namespace NCI.OCPL.Api.BestBets { /// - /// Represents a Best Best Match from the Search Engine + /// Represents a Best Bets Match from the Search Engine /// - [ElasticsearchType(RelationName = "synonyms")] public class BestBetsMatch { /// /// Gets or sets the name of the category for this Best Bet Match /// - [Text(Name = "category")] + [JsonPropertyName("category")] public string Category { get; set; } /// /// Gets or sets the content ID of the category of this match /// - [Text(Name = "contentid")] + [JsonPropertyName("contentid")] public string ContentID {get; set;} /// /// Gets or sets the synonym that was matched /// - [Text(Name = "synonym")] + [JsonPropertyName("synonym")] public string Synonym {get; set;} /// /// Gets or sets the two-character language code for this match /// - [Text(Name = "language")] + [JsonPropertyName("language")] public string Language {get; set;} /// /// Gets or sets whether this match is a negated one or not. /// - [Boolean(Name = "is_negated")] + [JsonPropertyName("is_negated")] public bool IsNegated {get; set;} /// /// Gets or sets whether this match is an exact one or not. /// - [Boolean(Name = "is_exact")] + [JsonPropertyName("is_exact")] public bool IsExact {get; set;} /// /// Gets or sets the number of tokens the ElasticSearch analyzer would /// return for this synonym. /// - [Number(Name = "tokencount")] + [JsonPropertyName("tokencount")] public int TokenCount {get; set;} } diff --git a/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj b/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj index 6bfe96a..a7adfdc 100644 --- a/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj +++ b/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj @@ -1,13 +1,12 @@ - netcoreapp8.0 + net8.0 true - - - + + diff --git a/src/NCI.OCPL.Api.BestBets/Program.cs b/src/NCI.OCPL.Api.BestBets/Program.cs index f767afa..84362a8 100644 --- a/src/NCI.OCPL.Api.BestBets/Program.cs +++ b/src/NCI.OCPL.Api.BestBets/Program.cs @@ -1,13 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - - using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using NCI.OCPL.Api.Common; diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs index d4d2833..c4c03c5 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs @@ -1,17 +1,10 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json; - -using NCI.OCPL.Api.BestBets; +using Elastic.Clients.Elasticsearch; using NCI.OCPL.Api.Common; @@ -23,7 +16,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsDisplayService : IBestBetsDisplayService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -33,7 +26,7 @@ public class ESBestBetsDisplayService : IBestBetsDisplayService /// The client to be used for connections /// The client to be used for connections /// The client to be used for connections - public ESBestBetsDisplayService(IElasticClient client, + public ESBestBetsDisplayService(ElasticsearchClient client, IOptions config, ILogger logger) { _elasticClient = client; @@ -80,22 +73,20 @@ public async Task GetBestBetForDisplay(string collection, strin throw new APIErrorException(500, $"Could not fetch category ID {categoryID}"); } - // If the API's response isn't valid, throw an error and return 500 status code. - if (!response.IsValid) + // The ES client treats "Not Found" and server errors as both "Not Found" and "Not Valid", + // so we also have to check the status code to determine what's really going on. + if (!response.Found && response.ApiCallDetails.HttpStatusCode == 404) { - throw new APIErrorException(500, "Errors occurred."); + throw new APIErrorException(404, "Category not found."); } - // If the API finds the category ID, return the resource. - if (response.Found && response.IsValid) - { - result = response.Source; - } - // If the API cannot find the category ID, throw an error and return 404 status code. - else if (!response.Found && response.IsValid) + // If the API's response isn't valid, throw an error and return 500 status code. + if (!response.IsValidResponse) { - throw new APIErrorException(404, "Category not found."); + throw new APIErrorException(500, "Errors occurred."); } + + result = response.Source; } else { diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs index ccfa1a8..9013dd7 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs @@ -1,17 +1,12 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json; - -using NCI.OCPL.Api.BestBets; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Cluster; using NCI.OCPL.Api.Common; @@ -23,7 +18,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsHealthService : IHealthCheckService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -33,7 +28,7 @@ public class ESBestBetsHealthService : IHealthCheckService /// The client to be used for connections /// The client to be used for connections /// The client to be used for connections - public ESBestBetsHealthService(IElasticClient client, + public ESBestBetsHealthService(ElasticsearchClient client, IOptions config, ILogger logger) { _elasticClient = client; @@ -75,17 +70,16 @@ private async Task IsHostHealthy(string alias) try { - //ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(hd => hd.Index(alias)); - ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(null, hd=> hd.Index(alias)); + HealthResponse response = await _elasticClient.Cluster.HealthAsync(new HealthRequest(alias)); - if (!response.IsValid) + if (!response.IsValidResponse) { _logger.LogError($"Error checking ElasticSearch health for {alias}."); - _logger.LogError($"Returned debug info: {response.DebugInformation}."); + _logger.LogError("Returned error reason: {reason}.", response.ElasticsearchServerError?.Error?.Reason); } else { - if (response.Status == Health.Green || response.Status == Health.Yellow) + if (response.Status == HealthStatus.Green || response.Status == HealthStatus.Yellow) { //This is the only condition that will return true return true; diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs index fe64e78..d35ec49 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs @@ -6,8 +6,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using NCI.OCPL.Api.Common; @@ -21,7 +21,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsMatchService : IBestBetsMatchService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private ITokenAnalyzerService _tokenAnalyzer; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -29,7 +29,7 @@ public class ESBestBetsMatchService : IBestBetsMatchService /// /// Creates a new instance of a ESBestBetsMatchService /// - public ESBestBetsMatchService(IElasticClient client, + public ESBestBetsMatchService(ElasticsearchClient client, ITokenAnalyzerService tokenAnalyzer, IOptions config, ILogger logger) @@ -45,7 +45,7 @@ public ESBestBetsMatchService(IElasticClient client, /// /// The search index to use /// The two-character language code to constrain the matches to - /// A term that have been cleaned of punctuation and special characters + /// A term that has been cleaned of punctuation and special characters /// An array of category ids public async Task GetMatches(string collection, string language, string cleanedTerm) { @@ -100,7 +100,7 @@ private string[] FilterValidCategories(IEnumerable matches, int n // For example, "Breast Cancer Treatment" would return the // Best Bets for "Breast Cancer" and "Breast Cancer Treatment". // However, as "Breast Cancer Treatment" is more specific, a - // BB editor has created a Negated synonyn of "Treatment" for + // BB editor has created a Negated synonym of "Treatment" for // the "Breast Cancer" category. So we should only show // "Breast Cancer Treatment" to the user. if (!excludedIDs.Contains(match.ContentID)) @@ -137,21 +137,37 @@ private string[] FilterValidCategories(IEnumerable matches, int n private async Task> GetSetOfMatchesAsync(string collection, string cleanedTerm, int searchTokenCount, string lang, int matchedTokenCount) { //This is the query - var matchQuery = new NumericRangeQuery { Field = "tokencount", LessThanOrEqualTo = matchedTokenCount } && - new TermQuery { Field = "is_exact", Value = false } && - new TermQuery { Field = "language", Value = lang } && - new MatchQuery { Field = "synonym", Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount } && - new TermQuery { Field = "record_type", Value = "synonyms" }; + var mustClauses = new List + { + new NumberRangeQuery(new Field("tokencount")) { Lte = matchedTokenCount }, + new TermQuery { Field = new Field("is_exact"), Value = false }, + new TermQuery { Field = new Field("language"), Value = lang }, + new MatchQuery { Field = new Field("synonym"), Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount.ToString() }, + new TermQuery { Field = new Field("record_type"), Value = "synonyms" } + }; + + Query matchQuery = new BoolQuery { Must = mustClauses }; if (searchTokenCount == matchedTokenCount) { //Add in exact match query too - matchQuery = matchQuery || - new TermQuery { Field = "tokencount", Value = matchedTokenCount } && - new TermQuery { Field = "is_exact", Value = true } && - new TermQuery { Field = "language", Value = lang } && - new MatchQuery { Field = "synonym", Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount } && - new TermQuery { Field = "record_type", Value = "synonyms"}; + var exactClauses = new List + { + new TermQuery { Field = new Field("tokencount"), Value = matchedTokenCount }, + new TermQuery { Field = new Field("is_exact"), Value = true }, + new TermQuery { Field = new Field("language"), Value = lang }, + new MatchQuery { Field = new Field("synonym"), Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount.ToString() }, + new TermQuery { Field = new Field("record_type"), Value = "synonyms" } + }; + + matchQuery = new BoolQuery + { + Should = new List + { + matchQuery, + new BoolQuery { Must = exactClauses } + } + }; } try @@ -160,7 +176,7 @@ private async Task> GetSetOfMatchesAsync(string colle this._bestbetsConfig.PreviewAliasName : this._bestbetsConfig.LiveAliasName; - var req = new SearchRequest(alias) + var req = new SearchRequest(alias) { Query = matchQuery, Size = 10000 //Make sure this more than the number of synonyms @@ -169,14 +185,14 @@ private async Task> GetSetOfMatchesAsync(string colle var response = await this._elasticClient.SearchAsync(req); //Test if response is valid - if (!response.IsValid) + if (!response.IsValidResponse) { _logger.LogError("Elasticsearch Response is Not Valid. Term '{0}'", cleanedTerm.Replace(Environment.NewLine, String.Empty)); - _logger.LogError("Returned debug info: {0}.", response.DebugInformation); + _logger.LogError("Returned error reason: {0}.", response.ElasticsearchServerError?.Error?.Reason); throw new APIErrorException(500, "Errors Occurred."); } - return response.Documents; + return response.Hits.Select(h => h.Source).Where(s => s != null)!; } catch (APIErrorException) diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs index 0d97495..af38268 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs @@ -1,13 +1,12 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; using NCI.OCPL.Api.Common; @@ -20,14 +19,14 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESTokenAnalyzerService : ITokenAnalyzerService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; /// /// Creates a new instance of a ESBestBetsMatchService /// - public ESTokenAnalyzerService(IElasticClient client, + public ESTokenAnalyzerService(ElasticsearchClient client, IOptions bestbetsConfig, ILogger logger) //Needs someway to get an IElasticClient { @@ -46,7 +45,7 @@ public async Task GetTokenCount(string collection, string term) { string[] ALLOWED_TOKEN_TYPES = { "", ""}; - AnalyzeResponse analyzeResponse; + AnalyzeIndexResponse analyzeResponse; string indexForAnalysis = (collection == "preview") ? _bestbetsConfig.PreviewAliasName : _bestbetsConfig.LiveAliasName; @@ -54,32 +53,32 @@ public async Task GetTokenCount(string collection, string term) try { analyzeResponse = await this._elasticClient.Indices.AnalyzeAsync( - a => a - .Index(indexForAnalysis) - .Analyzer("nostem") - .Text(term) + new AnalyzeIndexRequest(indexForAnalysis) + { + Analyzer = "nostem", + Text = new[] { term } + } ); } - catch(UnexpectedElasticsearchClientException ex) + catch(Exception ex) { - _logger.LogError(ex, "Error analyzing token count for term '{0}'. Reason: '{1}'. DebugInformation: {2}", - term.Replace(Environment.NewLine, String.Empty), - ex.FailureReason, ex.DebugInformation); + _logger.LogError(ex, "Error analyzing token count for term '{0}'.", term.Replace(Environment.NewLine, String.Empty)); _logger.LogInformation("Trying again for term '{0}", term); // Try again. (this is really just for when we run out of sockets) analyzeResponse = await this._elasticClient.Indices.AnalyzeAsync( - a => a - .Index(indexForAnalysis) - .Analyzer("nostem") - .Text(term) + new AnalyzeIndexRequest(indexForAnalysis) + { + Analyzer = "nostem", + Text = new[] { term } + } ); } - if (!analyzeResponse.IsValid) + if (!analyzeResponse.IsValidResponse) { _logger.LogError("Elasticsearch Response for GetTokenCount is Not Valid. Term '{0}'", term); - _logger.LogError("Returned debug info: {0}.", analyzeResponse.DebugInformation); + _logger.LogError("Returned error reason: {0}.", analyzeResponse.ElasticsearchServerError?.Error?.Reason); throw new APIErrorException(500, "Errors Occurred."); } diff --git a/src/NCI.OCPL.Api.BestBets/Startup.cs b/src/NCI.OCPL.Api.BestBets/Startup.cs index 95b45ef..2bea70f 100644 --- a/src/NCI.OCPL.Api.BestBets/Startup.cs +++ b/src/NCI.OCPL.Api.BestBets/Startup.cs @@ -1,26 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -using Nest; -using Elasticsearch.Net; - -using NSwag.AspNetCore; -using NJsonSchema; -using System.Reflection; -using NCI.OCPL.Api.Common; using NCI.OCPL.Api.BestBets.Services; +using NCI.OCPL.Api.Common; namespace NCI.OCPL.Api.BestBets { diff --git a/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj b/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj index 3208a2d..a019689 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj +++ b/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp8.0 + net8.0 @@ -15,16 +15,19 @@ - + - - - + + + + + + diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs index 01cac7e..cecde89 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs @@ -1,25 +1,14 @@ -using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Testing; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json.Linq; - using Moq; using Xunit; using NCI.OCPL.Api.Common; -using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets; using NCI.OCPL.Api.BestBets.Controllers; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; using NCI.OCPL.Api.BestBets.Tests.ESDisplayTestData; namespace NCI.OCPL.Api.BestBets.Tests @@ -36,7 +25,7 @@ public class BestBetsControllerTests }; [Fact] - public async void Get_Error_CollectionEmpty() + public async Task Get_Error_CollectionEmpty() { Mock displayService = new Mock(); Mock matchService = new Mock(); @@ -54,7 +43,7 @@ public async void Get_Error_CollectionEmpty() } [Fact] - public async void Get_Error_CollectionBad() + public async Task Get_Error_CollectionBad() { Mock displayService = new Mock(); Mock matchService = new Mock(); @@ -72,7 +61,7 @@ public async void Get_Error_CollectionBad() } [Fact] - public async void Get_Error_LanguageEmpty() + public async Task Get_Error_LanguageEmpty() { Mock displayService = new Mock(); Mock matchService = new Mock(); @@ -90,7 +79,7 @@ public async void Get_Error_LanguageEmpty() } [Fact] - public async void Get_Error_LanguageBad() + public async Task Get_Error_LanguageBad() { Mock displayService = new Mock(); Mock matchService = new Mock(); @@ -108,7 +97,7 @@ public async void Get_Error_LanguageBad() } [Fact] - public async void Get_Error_SearchTermBad() + public async Task Get_Error_SearchTermBad() { Mock displayService = new Mock(); Mock matchService = new Mock(); @@ -127,7 +116,7 @@ public async void Get_Error_SearchTermBad() [Theory, MemberData(nameof(XmlDeserializingData))] - public async void Get_EnglishTerm(string searchTerm, BaseDisplayTestData displayData) + public async Task Get_EnglishTerm(string searchTerm, BaseDisplayTestData displayData) { @@ -177,7 +166,7 @@ public async void Get_EnglishTerm(string searchTerm, BaseDisplayTestData display /// Verify that Status returns successfully when the health check service is healthy. /// [Fact] - public async void IsHealthy_Healthy() + public async Task IsHealthy_Healthy() { IBestBetsDisplayService displayService = null; IBestBetsMatchService matchService = null; @@ -209,7 +198,7 @@ private static T GetMockedHealthSvc( bool status) where T : class, IHealthChe /// Verify that Status fails for the unhealthy health check service. /// [Fact] - public async void IsHealthy_Unhealthy() + public async Task IsHealthy_Unhealthy() { IBestBetsDisplayService displayService = null; IBestBetsMatchService matchService = null; diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs index f472e50..d246f7f 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -19,7 +18,7 @@ public class ArrayComparer : IEqualityComparer /// public bool Equals(string[] x, string[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs index 78ef1dc..2139d16 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -14,12 +13,12 @@ public class IBestBetSynonymComparer : IEqualityComparer public bool Equals(IBestBetSynonym x, IBestBetSynonym y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - if (x == null && y== null) + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; @@ -28,7 +27,7 @@ public bool Equals(IBestBetSynonym x, IBestBetSynonym y) bool isEqual = x.IsExactMatch == y.IsExactMatch && x.Text == y.Text; - + return isEqual; } @@ -52,27 +51,27 @@ public class IBestBetCategoryComparer : IEqualityComparer public bool Equals(IBestBetCategory x, IBestBetCategory y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - if (x == null && y== null) + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; - } + } - bool isEqual = + bool isEqual = _displayComparer.Equals(x,y) //Handles ID, Name, Weight, HTML && x.Display == y.Display && x.IsExactMatch == y.IsExactMatch && x.Language == y.Language && AreSynonymListsEqual(x.ExcludeSynonyms, y.ExcludeSynonyms) && AreSynonymListsEqual(x.IncludeSynonyms, y.IncludeSynonyms); - - - return isEqual; + + + return isEqual; } /// @@ -82,17 +81,17 @@ public bool Equals(IBestBetCategory x, IBestBetCategory y) /// Synonym list 2 /// private bool AreSynonymListsEqual(IBestBetSynonym[] x, IBestBetSynonym[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - - if (x == null && y== null) + + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; - } + } //Generate a set of those values that are not in both lists. //if this is not 0, then there is an error. @@ -104,7 +103,7 @@ private bool AreSynonymListsEqual(IBestBetSynonym[] x, IBestBetSynonym[] y) { public int GetHashCode(IBestBetCategory obj) { int hash = 0; - hash ^= + hash ^= _displayComparer.GetHashCode(obj) //Handles ID, Name, Weight, HTML ^ obj.Display.GetHashCode() ^ obj.Language.GetHashCode() @@ -114,6 +113,6 @@ public int GetHashCode(IBestBetCategory obj) return hash; } - + } } \ No newline at end of file diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs index f84a3d7..1a13ce7 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -13,7 +12,7 @@ public class IBestBetDisplayComparer : IEqualityComparer { public bool Equals(IBestBetDisplay x, IBestBetDisplay y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) { @@ -65,7 +64,7 @@ public int GetHashCode(IBestBetDisplay obj) /// private bool AreParamArraysEqual(string[] x, string[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs index c259736..ff0bf76 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs @@ -1,24 +1,20 @@ using System; using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; -using Xunit; +using Elastic.Clients.Elasticsearch; using Moq; -using RichardSzalay.MockHttp; +using Xunit; -using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.Common; +using NCI.OCPL.Api.Common.Testing; -using Elasticsearch.Net; -using Nest; using NCI.OCPL.Api.BestBets.Services; using NCI.OCPL.Api.BestBets.Tests.ESDisplayTestData; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests.ESBestBetsDisplayServiceTests { @@ -38,27 +34,16 @@ public class GetBestBetForDisplayTests /// Test that URI for Elasticsearch is set up correctly. /// [Theory, MemberData(nameof(JsonData))] - public async void GetBestBetForDisplay_TestURISetup(BaseDisplayTestData data) + public async Task GetBestBetForDisplay_TestURISetup(BaseDisplayTestData data) { Uri esURI = null; - - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream("ESDisplayData/" + data.TestFilePath); - - res.StatusCode = 200; - - esURI = req.Uri; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + string responseBody = TestingTools.ReadTestFile("ESDisplayData/" + data.TestFilePath); + var settings = TestingElasticsearchClientSettingsFactory.Create( + responseBody, + 200, + details => esURI = details.Uri + ); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -86,20 +71,10 @@ public async void GetBestBetForDisplay_TestURISetup(BaseDisplayTestData data) /// Test failure to connect to and retrieve response from API. /// [Fact()] - public async void GetBestBetForDisplay_TestAPIConnectionFailure() + public async Task GetBestBetForDisplay_TestAPIConnectionFailure() { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - res.StatusCode = 500; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create("{}", 500); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -114,20 +89,10 @@ public async void GetBestBetForDisplay_TestAPIConnectionFailure() /// Test invalid response from API. /// [Fact()] - public async void GetBestBetForDisplay_TestInvalidResponse() + public async Task GetBestBetForDisplay_TestInvalidResponse() { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create("not-json", 200); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -144,9 +109,9 @@ public async void GetBestBetForDisplay_TestInvalidResponse() /// /// [Theory, MemberData(nameof(JsonData))] - public async void GetBestBetForDisplay_DataLoading(BaseDisplayTestData data) + public async Task GetBestBetForDisplay_DataLoading(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + ElasticsearchClient client = GetElasticClientWithData(data); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -162,9 +127,13 @@ public async void GetBestBetForDisplay_DataLoading(BaseDisplayTestData data) /// Test for handling API cannot find ID. /// [Theory, MemberData(nameof(NotFoundData))] - public async void GetBestBetForDisplay_IDNotFoundError(BaseDisplayTestData data) + public async Task GetBestBetForDisplay_IDNotFoundError(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + // This test needs the mock ES instance to return a 404 status, and therefore can't use the + // same GetElasticClientWithData method as the other tests. + string responseBody = TestingTools.ReadTestFile("ESDisplayData/" + data.TestFilePath); + var settings = TestingElasticsearchClientSettingsFactory.Create(responseBody, 404); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -179,9 +148,9 @@ public async void GetBestBetForDisplay_IDNotFoundError(BaseDisplayTestData data) /// Test for handling invalid ID. /// [Theory, MemberData(nameof(JsonData))] - public async void GetBestBetForDisplay_InvalidIDError(BaseDisplayTestData data) + public async Task GetBestBetForDisplay_InvalidIDError(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + ElasticsearchClient client = GetElasticClientWithData(data); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -192,24 +161,10 @@ public async void GetBestBetForDisplay_InvalidIDError(BaseDisplayTestData data) Assert.Equal(400, ex.HttpStatusCode); } - private IElasticClient GetElasticClientWithData(BaseDisplayTestData data) { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream("ESDisplayData/" + data.TestFilePath); - - res.StatusCode = 200; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); - - return client; + private ElasticsearchClient GetElasticClientWithData(BaseDisplayTestData data) { + string responseBody = TestingTools.ReadTestFile("ESDisplayData/" + data.TestFilePath); + var settings = TestingElasticsearchClientSettingsFactory.Create(responseBody, 200); + return new ElasticsearchClient(settings); } private IOptions GetMockOptions() diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs index c160d76..521e5c1 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs @@ -1,16 +1,12 @@ -using System; -using System.Collections.Generic; - -using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; -using Nest; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; using Xunit; +using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets.Services; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; -using NCI.OCPL.Api.BestBets.Tests.ESMatchTestData; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests { @@ -19,11 +15,9 @@ public class ESBestBetsHealthServiceTests : TestServiceBase [Theory] [InlineData("green")] [InlineData("yellow")] - public async void HealthStatus_Healthy(string datafile) + public async Task HealthStatus_Healthy(string datafile) { - ESHealthConnection connection = new ESHealthConnection(datafile); - - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthServiceFromFile($"ESHealthData/{datafile}.json", 200); bool isHealthy = await service.IsHealthy(); @@ -33,10 +27,9 @@ public async void HealthStatus_Healthy(string datafile) [Theory] [InlineData("red")] [InlineData("unexpected")] // i.e. "Unexpected color" - public async void HealthStatus_Unhealthy(string datafile) + public async Task HealthStatus_Unhealthy(string datafile) { - ESHealthConnection connection = new ESHealthConnection(datafile); - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthServiceFromFile($"ESHealthData/{datafile}.json", 200); bool isHealthy = await service.IsHealthy(); @@ -51,24 +44,24 @@ public async void HealthStatus_Unhealthy(string datafile) [Theory] [InlineData(404)] [InlineData(500)] - public async void HealthStatus_InvalidResponse(int httpStatus) + public async Task HealthStatus_InvalidResponse(int httpStatus) { - ESErrorConnection connection = new ESErrorConnection(httpStatus); - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthService("{}", httpStatus); bool res = await service.IsHealthy(); Assert.False(res); - } - private ESBestBetsHealthService GetHealthService(IConnection connection) + private ESBestBetsHealthService GetHealthServiceFromFile(string filename, int statusCode) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + string responseBody = TestingTools.ReadTestFile(filename); + return GetHealthService(responseBody, statusCode); + } - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + private ESBestBetsHealthService GetHealthService(string responseBody, int statusCode) + { + var settings = TestingElasticsearchClientSettingsFactory.Create(responseBody, statusCode); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs index baa12b8..9d7b488 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs @@ -1,30 +1,28 @@ -using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; -using Nest; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Xunit; using NCI.OCPL.Api.BestBets.Services; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; -using NCI.OCPL.Api.BestBets.Tests.ESMatchTestData; +using NCI.OCPL.Api.Common.Testing; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests { public class ESBestBetsMatchServiceTests : TestServiceBase { - public static IEnumerable GetMatchesData => new[] { // "pancoast" is a simple test as it only has 1 hit, 1 word, and 1 BB category. new object[] { "pancoast", "en", - new ESMatchConnection("pancoast"), - new ESMatchTokenizerConnection("pancoast"), + "pancoast", new string[] { "36012" } }, // "breast cancer" is more complicated, it has 1 hit, 2 words, and the BB category @@ -32,8 +30,7 @@ public class ESBestBetsMatchServiceTests : TestServiceBase new object[] { "breast cancer", "en", - new ESMatchConnection("breastcancer"), - new ESMatchTokenizerConnection("breastcancer"), + "breastcancer", new string[] { "36408" } }, // "breast cancer treatment" is more complicated, it has 1 hit, 3 words, and no results for last page. @@ -41,79 +38,93 @@ public class ESBestBetsMatchServiceTests : TestServiceBase new object[] { "breast cancer treatment", "en", - new ESMatchConnection("breastcancertreatment"), - new ESMatchTokenizerConnection("breastcancertreatment"), + "breastcancertreatment", new string[] { "36408" } }, // "seer stat" is a negated exact match test. SEER should not be returned new object[] { "seer stat", "en", - new ESMatchConnection("seerstat"), - new ESMatchTokenizerConnection("seerstat"), + "seerstat", new string[] { } }, // "seer stat fact sheet" is a test to make sure the "seer stat" exact match is not hit because - // we are not exactly matching the phrase "seet stat". Those search terms also match seer. + // we are not exactly matching the phrase "seer stat". Those search terms also match seer. new object[] { "seer stat fact sheet", "en", - new ESMatchConnection("seerstatfactsheet"), - new ESMatchTokenizerConnection("seerstatfactsheet"), + "seerstatfactsheet", new string[] { "36681" } } }; [Theory, MemberData(nameof(GetMatchesData))] - public async void GetMatches_Normal( + public async Task GetMatches_Normal( string searchTerm, string lang, - ESMatchConnection connection, - ESMatchTokenizerConnection tokenizerConn, + string responseFileBase, string[] expectedCategories ) { //Use real ES client, with mocked connection. - - ESTokenAnalyzerService tokenService = GetTokenizerService(tokenizerConn); - ESBestBetsMatchService service = GetMatchService(tokenService, connection); + ESTokenAnalyzerService tokenService = GetTokenizerService(responseFileBase); + ESBestBetsMatchService service = GetMatchService(tokenService, responseFileBase); string[] actualMatches = await service.GetMatches("live", lang, searchTerm); Assert.Equal(expectedCategories, actualMatches); } - private ESTokenAnalyzerService GetTokenizerService(IConnection connection) + private ESTokenAnalyzerService GetTokenizerService(string responseFileBase) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + string body = TestingTools.ReadTestFile($"ESMatchData/{responseFileBase}_analyze.json"); + var settings = TestingElasticsearchClientSettingsFactory.Create(body, 200); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); return new ESTokenAnalyzerService(client, config, new NullLogger()); } - private ESBestBetsMatchService GetMatchService(ESTokenAnalyzerService tokenService, IConnection connection) + private ESBestBetsMatchService GetMatchService(ESTokenAnalyzerService tokenService, string responseFileBase) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + string builder(PostData postData, JsonNode jsonBody) { + //Determine which round we are performing + //int numTokens = postObj["params"].matchedtokencount; + + //If this is one item the bool node will be the nested match + //if it is both exact and matches, then the bool node will + //be a should. This code is tightly matched to the built query + //in the implementation + int numTokens = -1; + + string value; + var boolNode = jsonBody["query"]["bool"]; + if (boolNode["should"] != null) { + value = boolNode["should"][0] + ["bool"]["must"][3] + ["match"]["synonym"] + ["minimum_should_match"].GetValue(); + } else { + value = boolNode["must"][3] + ["match"]["synonym"] + ["minimum_should_match"].GetValue(); + + } + numTokens = int.Parse(value); + + return TestingTools.ReadTestFile($"ESMatchData/{responseFileBase}_{numTokens}.json"); + } - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + var invoker = new DynamicInMemoryConnection(builder); + var settings = TestingElasticsearchClientSettingsFactory.Create(invoker); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); return new ESBestBetsMatchService(client, tokenService, config, new NullLogger()); } - - - } -} \ No newline at end of file +} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs index 80ab2e6..b90adb0 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs @@ -1,19 +1,20 @@ -using System; +#nullable enable +using System; using System.Collections.Generic; -using System.IO; -using System.Text; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; -using Nest; -using Elasticsearch.Net; -using Newtonsoft.Json.Linq; using Xunit; using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets.Services; - +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests { @@ -44,38 +45,24 @@ public class ESTokenAnalyzerServiceTests : TestServiceBase /// Verify the GetTokenCount() method knows how to handle responses /// from elastic search. /// - /// The search term to tokenizse. + /// The search term to tokenize. /// The simulated response from elasticsearch. /// The expected token count. /// [Theory, MemberData(nameof(GetTokenCountData))] - public async void GetTokenCount_Responses( + public async Task GetTokenCount_Responses( string searchTerm, object[] responseTokens, int expectedCount ) { + JsonObject resObject = new JsonObject(); + resObject["tokens"] = new JsonArray(responseTokens + .Select(responseToken => JsonSerializer.SerializeToNode(responseToken)) + .ToArray()); - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - - conn.RegisterRequestHandlerForType( - (req, res) => - { - JObject resObject = new JObject(); - resObject["tokens"] = JArray.FromObject(responseTokens); - byte[] byteArray = Encoding.UTF8.GetBytes(resObject.ToString()); - - res.Stream = new MemoryStream(byteArray); - res.StatusCode = 200; - } - ); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create(resObject.ToString(), 200); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs index 3ec1c07..b1cf2f8 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Options; using Moq; - - namespace NCI.OCPL.Api.BestBets.Tests { /// diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs deleted file mode 100644 index 4ee25bb..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs +++ /dev/null @@ -1,29 +0,0 @@ -using NCI.OCPL.Api.Common.Testing; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// Class used for mocking requests to Elasticsearch that return an error. - /// - /// - public class ESErrorConnection : ElasticsearchInterceptingConnection - { - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// HTTP status code to return - public ESErrorConnection(int testErrorCode) - { - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - // Health check is a GET request (e.g. https://localhost:9299/_cluster/health/bestbets?pretty) - // but for an error, all we care about is the response code, so we don't load a data file. - - res.StatusCode = testErrorCode; - }); - - } - } -} - diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs deleted file mode 100644 index b29c65e..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; - -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// Class used for mocking BestBet HealthCheck requests to Elasticsearch. This should be - /// used as the base class of test specific Connections object passed into an ElasticClient. - /// - /// - public class ESHealthConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// The prefix of the test files - public ESHealthConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - // Health check is a GET request (e.g. https://localhost:9299/_cluster/health/bestbets?pretty) - // so we don't need to do anything special, just load the data file. - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - - } - - private string GetTestFileName() - { - return string.Format("ESHealthData/{0}.json", TestFilePrefix); - } - } -} - diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs deleted file mode 100644 index 5aa7576..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// This class is a helper around the ElasticsearchInterceptingConntection to handle - /// analyzer responses for a BB HealthCheck Request. (Basically, this class only exists - /// because we need to have a TokenizerConnection in order to instantiate Match Service.) - /// - /// - public class ESHealthTokenizerConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchTokenizerConnection class - /// - /// The prefix of the test files - public ESHealthTokenizerConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - //I don't care about the request for this... for now. - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - } - - private string GetTestFileName() - { - return string.Format("ESHealthData/{0}_analyze.json", TestFilePrefix); - } - - } -} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs deleted file mode 100644 index bf904df..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESMatchTestData -{ - /// - /// Class used for mocking BestBet Match requests to Elasticsearch. This should be - /// used as the base class of test specific Connections object passed into an ElasticClient. - /// - /// - public class ESMatchConnection : ElasticsearchInterceptingConnection - { - - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// The prefix of the test files - public ESMatchConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType>((req, res) => - { - //Get the request parameters - dynamic postObj = this.GetRequestPost(req); - - //Determine which round we are performing - //int numTokens = postObj["params"].matchedtokencount; - - //If this is one item the bool node will be the nested match - //if it is both exact and matches, then the bool node will - //be a should. This code is tightly matched to the built query - //in the implementation - int numTokens = -1; - - var boolNode = postObj["query"]["bool"]; - if (boolNode["should"] != null) { - numTokens = boolNode["should"][0] - ["bool"]["must"][3] - ["match"]["synonym"] - ["minimum_should_match"].ToObject(); - } else { - numTokens = boolNode["must"][3] - ["match"]["synonym"] - ["minimum_should_match"].ToObject(); - } - - - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName(numTokens)); - - res.StatusCode = 200; - }); - - } - - private string GetTestFileName(int numTerms) - { - return string.Format("ESMatchData/{0}_{1}.json", TestFilePrefix, numTerms); - } - } -} \ No newline at end of file diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs deleted file mode 100644 index 1a4881a..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESMatchTestData -{ - /// - /// This class is a helper around the ElasticsearchInterceptingConntection to handle - /// analyzer responses for a BB Match Request - /// - /// - public class ESMatchTokenizerConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchTokenizerConnection class - /// - /// The prefix of the test files - public ESMatchTokenizerConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - //I don't care about the request for this... for now. - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - } - - private string GetTestFileName() - { - return string.Format("ESMatchData/{0}_analyze.json", TestFilePrefix); - } - - } -} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs index e7c9383..49774ab 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs @@ -1,7 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests.Util {