Skip to content

Commit 5a4b544

Browse files
fupelaqufupelaqu
committed
add support for "orphan" nested elements (nested elements not queried)
1 parent 0cf72b3 commit 5a4b544

File tree

10 files changed

+386
-25
lines changed

10 files changed

+386
-25
lines changed

‎es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,14 @@ import app.softnetwork.elastic.sql.function.aggregate.COUNT
55
import app.softnetwork.elastic.sql.function.geo.{Distance, Meters}
66
import app.softnetwork.elastic.sql.operator._
77
import app.softnetwork.elastic.sql.query._
8-
import com.sksamuel.elastic4s.ElasticApi
8+
import com.sksamuel.elastic4s.{ElasticApi, FetchSourceContext}
99
import com.sksamuel.elastic4s.ElasticApi._
1010
import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp
11-
import com.sksamuel.elastic4s.http.search.queries.QueryBuilderFn
1211
import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn
1312
import com.sksamuel.elastic4s.script.Script
1413
import com.sksamuel.elastic4s.script.ScriptType.Source
1514
import com.sksamuel.elastic4s.searches.aggs.{Aggregation, FilterAggregation}
16-
import com.sksamuel.elastic4s.searches.queries.Query
15+
import com.sksamuel.elastic4s.searches.queries.{InnerHit, Query}
1716
import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest}
1817
import com.sksamuel.elastic4s.searches.sort.FieldSort
1918

@@ -35,19 +34,6 @@ package object bridge {
3534
)
3635
).minScore(request.score)
3736

38-
def buildNestedQueryJson(root: NestedElement, rootQuery: Query): String = {
39-
val innerHits = root.raw
40-
val rootQueryJson = QueryBuilderFn.apply(rootQuery).string
41-
val ret = s"""{
42-
"nested": {
43-
"path": "${root.path}",
44-
"query": $rootQueryJson,
45-
"inner_hits": $innerHits
46-
}
47-
}"""
48-
ret
49-
}
50-
5137
implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = {
5238
import request._
5339
val notNestedBuckets = buckets.filterNot(_.nested)
@@ -74,8 +60,75 @@ package object bridge {
7460
val notNestedAggregations = aggregations.filterNot(_.nested)
7561
val nestedAggregations =
7662
aggregations.filter(_.nested).groupBy(_.nestedAgg.map(_.name).getOrElse(""))
63+
64+
val nestedWithoutCriteriaQuery: Option[Query] =
65+
NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match {
66+
case Nil => None
67+
case nestedTrees =>
68+
def nestedInner(n: NestedElement): InnerHit = {
69+
var inner = innerHits(n.innerHitsName)
70+
n.size match {
71+
case Some(s) =>
72+
inner = inner.from(0).size(s)
73+
case _ =>
74+
}
75+
if (n.sources.nonEmpty) {
76+
inner = inner.fetchSource(
77+
FetchSourceContext(
78+
fetchSource = true,
79+
includes = n.sources.toArray
80+
)
81+
)
82+
}
83+
inner
84+
}
85+
86+
def buildNestedQuery(n: NestedElement): Query = {
87+
val children = n.children
88+
if (children.nonEmpty) {
89+
val innerQueries = children.map(child => buildNestedQuery(child))
90+
val combinedQuery = if (innerQueries.size == 1) {
91+
innerQueries.head
92+
} else {
93+
must(innerQueries)
94+
}
95+
nestedQuery(
96+
n.path,
97+
combinedQuery
98+
) /*.scoreMode(ScoreMode.None)*/
99+
.inner(
100+
nestedInner(n)
101+
)
102+
} else {
103+
nestedQuery(
104+
n.path,
105+
matchAllQuery()
106+
) /*.scoreMode(ScoreMode.None)*/
107+
.inner(
108+
nestedInner(n)
109+
)
110+
}
111+
}
112+
113+
if (nestedTrees.size == 1) {
114+
Some(buildNestedQuery(nestedTrees.head))
115+
} else {
116+
val innerQueries = nestedTrees.map(nested => buildNestedQuery(nested))
117+
Some(boolQuery().filter(innerQueries))
118+
}
119+
}
120+
77121
var _search: SearchRequest = search("") query {
78-
where.flatMap(_.criteria.map(_.asQuery())).getOrElse(matchAllQuery())
122+
where.flatMap(_.criteria.map(_.asQuery())) match {
123+
case Some(c) =>
124+
val baseQuery = c
125+
nestedWithoutCriteriaQuery match {
126+
case Some(nc) => boolQuery().filter(baseQuery, nc)
127+
case _ => baseQuery
128+
}
129+
case _ =>
130+
nestedWithoutCriteriaQuery.getOrElse(matchAllQuery())
131+
}
79132
} sourceFiltering (fields, excludes)
80133

81134
_search = if (nestedAggregations.nonEmpty) {

‎es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3317,4 +3317,108 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
33173317
.replaceAll("DateTimeFormatter", " DateTimeFormatter")
33183318
}
33193319

3320+
it should "handle nested without criteria" in {
3321+
val select: ElasticSearchRequest =
3322+
SQLQuery(nestedWithoutCriteria)
3323+
val query = select.query
3324+
println(query)
3325+
query shouldBe
3326+
"""{
3327+
| "query": {
3328+
| "bool": {
3329+
| "filter": [
3330+
| {
3331+
| "bool": {
3332+
| "filter": [
3333+
| {
3334+
| "script": {
3335+
| "script": {
3336+
| "lang": "painless",
3337+
| "source": "def left = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); left == null ? false : left < ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()"
3338+
| }
3339+
| }
3340+
| }
3341+
| ]
3342+
| }
3343+
| },
3344+
| {
3345+
| "nested": {
3346+
| "path": "comments",
3347+
| "query": {
3348+
| "nested": {
3349+
| "path": "comments.replies",
3350+
| "query": {
3351+
| "match_all": {}
3352+
| },
3353+
| "inner_hits": {
3354+
| "name": "matched_replies",
3355+
| "from": 0,
3356+
| "_source": {
3357+
| "includes": [
3358+
| "reply_author",
3359+
| "reply_text"
3360+
| ]
3361+
| },
3362+
| "size": 5
3363+
| }
3364+
| }
3365+
| },
3366+
| "inner_hits": {
3367+
| "name": "matched_comments",
3368+
| "from": 0,
3369+
| "_source": {
3370+
| "includes": [
3371+
| "author",
3372+
| "comments"
3373+
| ]
3374+
| },
3375+
| "size": 5
3376+
| }
3377+
| }
3378+
| }
3379+
| ]
3380+
| }
3381+
| },
3382+
| "from": 0,
3383+
| "size": 5,
3384+
| "_source": true
3385+
|}""".stripMargin
3386+
.replaceAll("\\s+", "")
3387+
.replaceAll("\\s+", "")
3388+
.replaceAll("\\s+", "")
3389+
.replaceAll("defv", " def v")
3390+
.replaceAll("defa", "def a")
3391+
.replaceAll("defe", "def e")
3392+
.replaceAll("defl", "def l")
3393+
.replaceAll("def_", "def _")
3394+
.replaceAll("=_", " = _")
3395+
.replaceAll(",_", ", _")
3396+
.replaceAll(",\\(", ", (")
3397+
.replaceAll("if\\(", "if (")
3398+
.replaceAll(">=", " >= ")
3399+
.replaceAll("=\\(", " = (")
3400+
.replaceAll(":\\(", " : (")
3401+
.replaceAll(",(\\d)", ", $1")
3402+
.replaceAll("\\?", " ? ")
3403+
.replaceAll(":null", " : null")
3404+
.replaceAll("null:", "null : ")
3405+
.replaceAll("return", " return ")
3406+
.replaceAll(";", "; ")
3407+
.replaceAll("; if", ";if")
3408+
.replaceAll("==", " == ")
3409+
.replaceAll("\\+", " + ")
3410+
.replaceAll(">(\\d)", " > $1")
3411+
.replaceAll("=(\\d)", "= $1")
3412+
.replaceAll("<", " < ")
3413+
.replaceAll("!=", " != ")
3414+
.replaceAll("&&", " && ")
3415+
.replaceAll("\\|\\|", " || ")
3416+
.replaceAll("(\\d)=", "$1 = ")
3417+
.replaceAll(",params", ", params")
3418+
.replaceAll("GeoPoint", " GeoPoint")
3419+
.replaceAll("lat,arg", "lat, arg")
3420+
.replaceAll("false:", "false : ")
3421+
.replaceAll("DateTimeFormatter", " DateTimeFormatter")
3422+
}
3423+
33203424
}

‎sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ import app.softnetwork.elastic.sql.query._
88

99
import com.sksamuel.elastic4s.ElasticApi
1010
import com.sksamuel.elastic4s.ElasticApi._
11+
import com.sksamuel.elastic4s.requests.common.FetchSourceContext
1112
import com.sksamuel.elastic4s.requests.script.Script
1213
import com.sksamuel.elastic4s.requests.script.ScriptType.Source
1314
import com.sksamuel.elastic4s.requests.searches.aggs.{Aggregation, FilterAggregation}
14-
import com.sksamuel.elastic4s.requests.searches.queries.Query
15+
import com.sksamuel.elastic4s.requests.searches.queries.{InnerHit, Query}
1516
import com.sksamuel.elastic4s.requests.searches.sort.FieldSort
1617
import com.sksamuel.elastic4s.requests.searches.{
1718
MultiSearchRequest,
@@ -63,10 +64,78 @@ package object bridge {
6364
val notNestedAggregations = aggregations.filterNot(_.nested)
6465
val nestedAggregations =
6566
aggregations.filter(_.nested).groupBy(_.nestedAgg.map(_.name).getOrElse(""))
67+
68+
val nestedWithoutCriteriaQuery: Option[Query] =
69+
NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match {
70+
case Nil => None
71+
case nestedTrees =>
72+
def nestedInner(n: NestedElement): InnerHit = {
73+
var inner = innerHits(n.innerHitsName)
74+
n.size match {
75+
case Some(s) =>
76+
inner = inner.from(0).size(s)
77+
case _ =>
78+
}
79+
if (n.sources.nonEmpty) {
80+
inner = inner.fetchSource(
81+
FetchSourceContext(
82+
fetchSource = true,
83+
includes = n.sources.toArray
84+
)
85+
)
86+
}
87+
inner
88+
}
89+
90+
def buildNestedQuery(n: NestedElement): Query = {
91+
val children = n.children
92+
if (children.nonEmpty) {
93+
val innerQueries = children.map(child => buildNestedQuery(child))
94+
val combinedQuery = if (innerQueries.size == 1) {
95+
innerQueries.head
96+
} else {
97+
must(innerQueries)
98+
}
99+
nestedQuery(
100+
n.path,
101+
combinedQuery
102+
) /*.scoreMode(ScoreMode.None)*/
103+
.inner(
104+
nestedInner(n)
105+
)
106+
} else {
107+
nestedQuery(
108+
n.path,
109+
matchAllQuery()
110+
) /*.scoreMode(ScoreMode.None)*/
111+
.inner(
112+
nestedInner(n)
113+
)
114+
}
115+
}
116+
117+
if (nestedTrees.size == 1) {
118+
Some(buildNestedQuery(nestedTrees.head))
119+
} else {
120+
val innerQueries = nestedTrees.map(nested => buildNestedQuery(nested))
121+
Some(boolQuery().filter(innerQueries))
122+
}
123+
}
124+
66125
var _search: SearchRequest = search("") query {
67-
where.flatMap(_.criteria.map(_.asQuery())).getOrElse(matchAllQuery())
126+
where.flatMap(_.criteria.map(_.asQuery())) match {
127+
case Some(c) =>
128+
val baseQuery = c
129+
nestedWithoutCriteriaQuery match {
130+
case Some(nc) => boolQuery().filter(baseQuery, nc)
131+
case _ => baseQuery
132+
}
133+
case _ =>
134+
nestedWithoutCriteriaQuery.getOrElse(matchAllQuery())
135+
}
68136
} sourceFiltering (fields, excludes)
69137

138+
70139
_search = if (nestedAggregations.nonEmpty) {
71140
_search aggregations {
72141
nestedAggregations.map { case (nested, aggs) =>
@@ -100,7 +169,6 @@ package object bridge {
100169
_search = notNestedAggregations match {
101170
case Nil => _search
102171
case _ => _search aggregations {
103-
val first = notNestedAggregations.head
104172
val aggregationDirections: Map[String, SortOrder] = notNestedAggregations
105173
.filter(_.direction.isDefined)
106174
.map(agg => agg.agg.name -> agg.direction.get)

0 commit comments

Comments
 (0)