(#78) Elasticsearch 8 update

integration-tests/README.md

@@ -12,13 +12,13 @@
 
 ## Dependencies
 
-The integration tests download test data from a sepcific commit hash in the [bestbets-content GitHub repository](https://github.com/nciocpl/bestbets-content). To retrieve data from a different commit, edit `bin/load-integration-data.sh` and modify the value of the `BESTBETS_CONTENT_COMMIT` environment variable.
+The integration tests download test data from a specific commit hash in the [bestbets-content GitHub repository](https://github.com/nciocpl/bestbets-content). To retrieve data from a different commit, edit `bin/load-integration-data.sh` and modify the value of the `BESTBETS_CONTENT_COMMIT` environment variable.
 
 the integration data is loaded by referencing the [bestbets-loader](https://github.com/nciocpl/bestbets-loader) as a git submodule. It is downloaded, installed, and executed via `load-integration-data.sh`. To update the loader's version, go to the `bin/bestbets-loader` directory, fetch and checkout the newer version, and commit the change.  (See also: [the submodule reference in the git manual](https://git-scm.com/book/en/v2/Git-Tools-Submodules).)
 
 ## Notes
 * [Docs for understanding how to run Karate standalone](https://github.com/intuit/karate/blob/6de466bdcf105d72450a40cf31b8adb5c043037d/karate-netty/README.md#standalone-jar)
    * Specifically this has to do with the magic naming of the logging config which is really why I am posting this here!
 * We have docker for dev testing because ES will no longer run on higher Java versions, this is the easiest way to get it up and running.
-* .NET running locally on a Mac cannot talk to ES because of how NEST always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac.
+* .NET running locally on a Mac cannot talk to ES because of how the client always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac.
 * You need to use the `--force-recreate` option to `docker-compose up` or run `docker-compose rm` after shutting down the cluster. If the elasticsearch container is not removed, it keeps its data, and any restarts will leave the cluster in a bad state.

integration-tests/docker-bestbets-api/api/Dockerfile

@@ -4,8 +4,8 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0
 # Get the NIH Root certificates and install them so we work on VPN
 # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx
 RUN mkdir /usr/local/share/ca-certificates/nih \
-    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \
-    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-CA-1A%281%29.crt \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-ROOT-1A.cer \
     && update-ca-certificates
 
 WORKDIR /app

integration-tests/docker-bestbets-api/docker-compose.yml

@@ -9,8 +9,11 @@ services:
         environment:
             - discovery.type=single-node
             - ES_JAVA_OPTS=-Xms750m -Xmx750m
-            # Turn off warnings about not using HTTPS.
+            # Turn security settings off for testing. (Allows http instead of https.)
             - xpack.security.enabled=false
+            - xpack.security.transport.ssl.enabled=false
+            - xpack.security.http.ssl.enabled=false
+            - http.host=0.0.0.0
         ## These exposed ports are for debugging only. .NET +
         ## Docker + MacOS == bad scene. (.NET always wants to
         ## use the hosts name, and on a mac that is actually
@@ -41,6 +44,7 @@ services:
             - ../../integration-tests/docker-bestbets-api/api/runtime/hosting.json:/app/src/NCI.OCPL.Api.BestBets/hosting.json
             # Use the user's existing GitHub credentials
             - ~/.nuget/NuGet/NuGet.Config:/root/.nuget/NuGet/NuGet.Config
+
         ## NOTE: This does NOT mean that this machine will wait
         ## for elasticsearch to be running, just that the
         ## elasticsearch container should be running first.

integration-tests/docker-bestbets-api/elasticsearch/Dockerfile

@@ -1,9 +1,12 @@
-FROM elasticsearch:7.17.28
+FROM elasticsearch:8.19.12
+
+USER root
 
 # Get the NIH Root certificates and install them so we work on VPN
 # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx
 RUN mkdir /usr/local/share/ca-certificates/nih \
-    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \
-    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-CA-1A%281%29.crt \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt http://nihdpkicrl.nih.gov/certchains/NIH-DPKI-ROOT-1A.cer \
     && update-ca-certificates
 
+USER elasticsearch

src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs

@@ -8,9 +8,6 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 
-using Nest;
-using Elasticsearch.Net;
-
 using NCI.OCPL.Api.Common;
 
 namespace NCI.OCPL.Api.BestBets.Controllers
@@ -91,7 +88,7 @@ public async Task<IBestBetDisplay[]> Get(string collection, string language, str
             if (String.IsNullOrWhiteSpace(term))
                 throw new APIErrorException(400, "You must supply a search term");
 
-            // Term comes from from a catch-all parameter, so make sure it's been decoded.
+            // Term comes from a catch-all parameter, so make sure it's been decoded.
             term = WebUtility.UrlDecode(term);
 
             // Step 1. Remove Punctuation

src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs

@@ -1,4 +1,3 @@
-
 using System.Threading.Tasks;
 
 namespace NCI.OCPL.Api.BestBets

src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsMatchService.cs

@@ -8,11 +8,11 @@ namespace NCI.OCPL.Api.BestBets
     public interface IBestBetsMatchService
     {
         /// <summary>
-        /// Gets a list of the BestBet Category IDs that matched our term 
-        /// </summary>        
+        /// Gets a list of the BestBet Category IDs that matched our term
+        /// </summary>
         /// <param name="collection">The collection to use. This will be 'live' or 'preview'.</param>
         /// <param name="language">The two-character language code to constrain the matches to</param>
-        /// <param name="cleanedTerm">A term that have been cleaned of punctuation and special characters</param>
+        /// <param name="cleanedTerm">A term that has been cleaned of punctuation and special characters</param>
         /// <returns>An array of category ids</returns>
         Task<string[]> GetMatches(string collection, string language, string cleanedTerm);
     }

src/NCI.OCPL.Api.BestBets/Interfaces/IESSearchOptions.cs

@@ -17,7 +17,7 @@ public interface IESSearchOptions
          string Username {get; set;}
 
          /// <summary>
-         /// Gets and sets the Username for authenticating to the ES server
+         /// Gets and sets the Password for authenticating to the ES server
          /// </summary>
          string Password {get; set;}
     }

src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs

@@ -1,5 +1,4 @@
-
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 
 namespace NCI.OCPL.Api.BestBets
 {

src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 
 namespace NCI.OCPL.Api.BestBets
 {

src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs

@@ -1,36 +1,34 @@
-using System;
-using Nest;
+using System.Text.Json.Serialization;
 
 namespace NCI.OCPL.Api.BestBets
 {
     /// <summary>
     /// Represents Display information about a Best Bet
     /// </summary>
-    [ElasticsearchType(RelationName = "categorydisplay")]
     public class BestBetsCategoryDisplay : IBestBetDisplay
     {
         /// <summary>
         /// Gets or sets the name of the category for this Best Bet Match
         /// </summary>
-        [Text(Name = "name")]
+        [JsonPropertyName("name")]
         public string Name { get; set; }
 
         /// <summary>
         /// Gets or sets the content ID of the category of this match
         /// </summary>
-        [Text(Name = "contentid")]
+        [JsonPropertyName("contentid")]
         public string ID { get; set; }
 
         /// <summary>
         /// Gets or sets the HTML for display for this category
         /// </summary>
-        [Text(Name = "content")]
+        [JsonPropertyName("content")]
         public string HTML { get; set; }
 
         /// <summary>
         /// Gets the weight of this category to determine ordering on display
         /// </summary>
-        [Text(Name = "weight")]
+        [JsonPropertyName("weight")]
         public int Weight { get; set; }
 
         /// <summary>

src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs

@@ -1,56 +1,53 @@
-using System;
-
-using Nest;
+using System.Text.Json.Serialization;
 
 namespace NCI.OCPL.Api.BestBets
 {
     /// <summary>
-    /// Represents a Best Best Match from the Search Engine
+    /// Represents a Best Bets Match from the Search Engine
     /// </summary>
-    [ElasticsearchType(RelationName = "synonyms")]
     public class BestBetsMatch
     {
         /// <summary>
         /// Gets or sets the name of the category for this Best Bet Match
         /// </summary>
-        [Text(Name = "category")]
+        [JsonPropertyName("category")]
         public string Category { get; set; }
 
         /// <summary>
         /// Gets or sets the content ID of the category of this match
         /// </summary>
-        [Text(Name = "contentid")]
+        [JsonPropertyName("contentid")]
         public string ContentID {get; set;}
 
         /// <summary>
         /// Gets or sets the synonym that was matched
         /// </summary>
-        [Text(Name = "synonym")]
+        [JsonPropertyName("synonym")]
         public string Synonym {get; set;}
 
         /// <summary>
         /// Gets or sets the two-character language code for this match
         /// </summary>
-        [Text(Name = "language")]
+        [JsonPropertyName("language")]
         public string Language {get; set;}
 
         /// <summary>
         /// Gets or sets whether this match is a negated one or not.
         /// </summary>
-        [Boolean(Name = "is_negated")]
+        [JsonPropertyName("is_negated")]
         public bool IsNegated {get; set;}
 
         /// <summary>
         /// Gets or sets whether this match is an exact one or not.
         /// </summary>
-        [Boolean(Name = "is_exact")]
+        [JsonPropertyName("is_exact")]
         public bool IsExact {get; set;}
 
         /// <summary>
         /// Gets or sets the number of tokens the ElasticSearch analyzer would
         /// return for this synonym.
         /// </summary>
-        [Number(Name = "tokencount")]
+        [JsonPropertyName("tokencount")]
         public int TokenCount {get; set;}
 
     }

src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj

@@ -1,13 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp8.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NCI.OCPL.Api.Common" Version="3.0.1" />
-    <PackageReference Include="NSwag.AspNetCore" Version="13.20.*" />
-    <PackageReference Include="NEST" Version="7.9.*" />
+    <PackageReference Include="NCI.OCPL.Api.Common" Version="4.0.0" />
+    <PackageReference Include="Elastic.Clients.Elasticsearch" Version="8.19.12" />
   </ItemGroup>
 </Project>

src/NCI.OCPL.Api.BestBets/Program.cs

@@ -1,13 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
-
-
 using Microsoft.AspNetCore;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 
 using NCI.OCPL.Api.Common;

src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs

@@ -1,17 +1,10 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
-using Elasticsearch.Net;
-using Nest;
-
-using Newtonsoft.Json;
-
-using NCI.OCPL.Api.BestBets;
+using Elastic.Clients.Elasticsearch;
 using NCI.OCPL.Api.Common;
 
 
@@ -23,7 +16,7 @@ namespace NCI.OCPL.Api.BestBets.Services
     /// </summary>
     public class ESBestBetsDisplayService : IBestBetsDisplayService
     {
-        private IElasticClient _elasticClient;
+        private ElasticsearchClient _elasticClient;
         private CGBBIndexOptions _bestbetsConfig;
         private readonly ILogger<ESBestBetsDisplayService> _logger;
 
@@ -33,7 +26,7 @@ public class ESBestBetsDisplayService : IBestBetsDisplayService
         /// <param name="client">The client to be used for connections</param>
         /// <param name="config">The client to be used for connections</param>
         /// <param name="logger">The client to be used for connections</param>
-        public ESBestBetsDisplayService(IElasticClient client,
+        public ESBestBetsDisplayService(ElasticsearchClient client,
             IOptions<CGBBIndexOptions> config,
             ILogger<ESBestBetsDisplayService> logger) {
             _elasticClient = client;
@@ -80,22 +73,20 @@ public async Task<IBestBetDisplay> GetBestBetForDisplay(string collection, strin
                     throw new APIErrorException(500, $"Could not fetch category ID {categoryID}");
                 }
 
-                // If the API's response isn't valid, throw an error and return 500 status code.
-                if (!response.IsValid)
+                // The ES client treats "Not Found" and server errors as both "Not Found" and "Not Valid",
+                // so we also have to check the status code to determine what's really going on.
+                if (!response.Found && response.ApiCallDetails.HttpStatusCode == 404)
                 {
-                    throw new APIErrorException(500, "Errors occurred.");
+                    throw new APIErrorException(404, "Category not found.");
                 }
 
-                // If the API finds the category ID, return the resource.
-                if (response.Found && response.IsValid)
-                {
-                    result = response.Source;
-                }
-                // If the API cannot find the category ID, throw an error and return 404 status code.
-                else if (!response.Found && response.IsValid)
+                // If the API's response isn't valid, throw an error and return 500 status code.
+                if (!response.IsValidResponse)
                 {
-                    throw new APIErrorException(404, "Category not found.");
+                    throw new APIErrorException(500, "Errors occurred.");
                 }
+
+                result = response.Source;
             }
             else
             {

src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs

@@ -1,17 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
-using Elasticsearch.Net;
-using Nest;
-
-using Newtonsoft.Json;
-
-using NCI.OCPL.Api.BestBets;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.Cluster;
 using NCI.OCPL.Api.Common;
 
 
@@ -23,7 +18,7 @@ namespace NCI.OCPL.Api.BestBets.Services
     /// </summary>
     public class ESBestBetsHealthService : IHealthCheckService
     {
-        private IElasticClient _elasticClient;
+        private ElasticsearchClient _elasticClient;
         private CGBBIndexOptions _bestbetsConfig;
         private readonly ILogger<ESBestBetsHealthService> _logger;
 
@@ -33,7 +28,7 @@ public class ESBestBetsHealthService : IHealthCheckService
         /// <param name="client">The client to be used for connections</param>
         /// <param name="config">The client to be used for connections</param>
         /// <param name="logger">The client to be used for connections</param>
-        public ESBestBetsHealthService(IElasticClient client,
+        public ESBestBetsHealthService(ElasticsearchClient client,
             IOptions<CGBBIndexOptions> config,
             ILogger<ESBestBetsHealthService> logger) {
             _elasticClient = client;
@@ -75,17 +70,16 @@ private async Task<bool> IsHostHealthy(string alias)
 
             try
             {
-                //ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(hd => hd.Index(alias));
-                ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(null, hd=> hd.Index(alias));
+                HealthResponse response = await _elasticClient.Cluster.HealthAsync(new HealthRequest(alias));
 
-                if (!response.IsValid)
+                if (!response.IsValidResponse)
                 {
                     _logger.LogError($"Error checking ElasticSearch health for {alias}.");
-                    _logger.LogError($"Returned debug info: {response.DebugInformation}.");
+                    _logger.LogError("Returned error reason: {reason}.", response.ElasticsearchServerError?.Error?.Reason);
                 }
                 else
                 {
-                    if (response.Status == Health.Green || response.Status == Health.Yellow)
+                    if (response.Status == HealthStatus.Green || response.Status == HealthStatus.Yellow)
                     {
                         //This is the only condition that will return true
                         return true;