Skip to content

feat: PostgreSQL-backed search/document storage as ElasticSearch alternative #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pandor4u wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat: PostgreSQL-backed search/document storage as ElasticSearch alternative #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2553448
feat: PostgreSQL-backed search/document storage as ElasticSearch alte…
pandor4u Apr 10, 2026
84f7c48
Improve PostgreSQL search backend: dedicated table routing, ts_headli…
pandor4u Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions framework/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ dependencies {
testImplementation 'org.junit.platform:junit-platform-suite:6.0.1'
// junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests
testImplementation 'org.junit.jupiter:junit-jupiter-api:6.0.1'
// junit-jupiter-engine required to execute @Test-annotated methods via JUnit Platform
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:6.0.1'
// Spock Framework
testImplementation platform('org.spockframework:spock-bom:2.4-groovy-5.0') // Apache 2.0
testImplementation 'org.spockframework:spock-core:2.4-groovy-5.0' // Apache 2.0
Expand Down Expand Up @@ -201,6 +203,7 @@ test {

dependsOn cleanTest
include '**/*MoquiSuite.class'
include '**/*PostgresSearchSuite.class'

systemProperty 'moqui.runtime', '../runtime'
systemProperty 'moqui.conf', 'conf/MoquiDevConf.xml'
Expand Down
115 changes: 115 additions & 0 deletions framework/entity/SearchEntities.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
This software is in the public domain under CC0 1.0 Universal plus a
Grant of Patent License.

To the extent possible under law, the author(s) have dedicated all
copyright and related and neighboring rights to this software to the
public domain worldwide. This software is distributed without any
warranty.

You should have received a copy of the CC0 Public Domain Dedication
along with this software (see the LICENSE.md file). If not, see
<http://creativecommons.org/publicdomain/zero/1.0/>.
-->
<!--
PostgreSQL-backed search and logging entities.
These tables are created by PostgresElasticClient.initSchema() at startup when type=postgres is configured.
Entity definitions here are provided for Moqui entity framework access (queries, etc) — they reference the
same underlying tables. The actual CREATE TABLE SQL includes JSONB columns and GIN/BRIN indexes that go
beyond what Moqui entities can express, so the DDL is managed by PostgresElasticClient directly.
-->
<entities xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="http://moqui.org/xsd/entity-definition-3.xsd">

<!-- ======================================================
Postgres Search — Document Store (replaces ES indexes)
====================================================== -->

<!-- moqui_search_index tracks index metadata (equivalent to ES index aliases + mappings storage) -->
<!-- authorize-skip="create" allows the internal PostgresElasticClient to create records without
per-user authorization. These entities should NOT be exposed via entity-level REST APIs
without additional access controls. -->
<entity entity-name="SearchIndex" package="moqui.search" table-name="moqui_search_index"
group="transactional" use="configuration" cache="never" authorize-skip="create">
<field name="indexName" type="text-medium" is-pk="true"/>
<field name="aliasName" type="text-medium"/>
<field name="docType" type="text-medium"/>
<!-- mapping and settings stored as JSON text; actual column is JSONB in PostgreSQL -->
<field name="mappingJson" type="text-very-long" column-name="mapping"/>
<field name="settingsJson" type="text-very-long" column-name="settings"/>
<field name="createdStamp" type="date-time" default="ec.user.nowTimestamp"/>
</entity>

<!-- moqui_document is the main document store for DataDocuments indexed for search -->
<entity entity-name="SearchDocument" package="moqui.search" table-name="moqui_document"
group="transactional" use="transactional" cache="never" authorize-skip="create">
<field name="indexName" type="text-medium" is-pk="true"/>
<field name="documentId" type="text-medium" is-pk="true" column-name="doc_id"/>
<field name="docType" type="text-medium"/>
<!-- document stored as JSON text; actual column is JSONB in PostgreSQL with GIN index -->
<field name="documentJson" type="text-very-long" column-name="document"/>
<!-- content_text is the extracted text for full-text search; content_tsv is a GENERATED ALWAYS AS tsvector column — not mapped here since Moqui can't express it, but it exists in the DB -->
<field name="contentText" type="text-very-long" column-name="content_text"/>
<field name="createdStamp" type="date-time" default="ec.user.nowTimestamp" column-name="created_stamp"/>
<field name="updatedStamp" type="date-time" column-name="updated_stamp"/>
<index name="SRCH_DOC_TYPE" unique="false"><index-field name="docType"/></index>
</entity>

<!-- ======================================================
Postgres Search — Application Log (replaces moqui_logs ES index)
====================================================== -->

<entity entity-name="SearchLog" package="moqui.search" table-name="moqui_logs"
group="transactional" use="transactional" cache="never"
sequence-bank-size="100" authorize-skip="create">
<field name="logId" type="number-integer" is-pk="true" column-name="log_id"/>
<field name="logTimestamp" type="date-time" column-name="log_timestamp"/>
<field name="logLevel" type="text-short" column-name="log_level"/>
<field name="threadName" type="text-short" column-name="thread_name"/>
<field name="threadId" type="number-integer" column-name="thread_id"/>
<field name="threadPriority" type="number-integer" column-name="thread_priority"/>
<field name="loggerName" type="text-medium" column-name="logger_name"/>
<field name="message" type="text-very-long"/>
<field name="sourceHost" type="text-short" column-name="source_host"/>
<field name="userId" type="id" column-name="user_id"/>
<field name="visitorId" type="id" column-name="visitor_id"/>
<!-- mdc and thrown stored as JSON text; actual columns are JSONB in PostgreSQL -->
<field name="mdcJson" type="text-long" column-name="mdc"/>
<field name="thrownJson" type="text-very-long" column-name="thrown"/>
<index name="MQLOGS_TS" unique="false"><index-field name="logTimestamp"/></index>
<index name="MQLOGS_LVL" unique="false"><index-field name="logLevel"/></index>
</entity>

<!-- ======================================================
Postgres Search — HTTP Request Log (replaces moqui_http_log ES index)
====================================================== -->

<entity entity-name="SearchHttpLog" package="moqui.search" table-name="moqui_http_log"
group="transactional" use="transactional" cache="never"
sequence-bank-size="100" authorize-skip="create">
<field name="logId" type="number-integer" is-pk="true" column-name="log_id"/>
<field name="logTimestamp" type="date-time" column-name="log_timestamp"/>
<field name="remoteIp" type="text-short" column-name="remote_ip"/>
<field name="remoteUser" type="text-short" column-name="remote_user"/>
<field name="serverIp" type="text-short" column-name="server_ip"/>
<field name="contentType" type="text-medium" column-name="content_type"/>
<field name="requestMethod" type="text-short" column-name="request_method"/>
<field name="requestScheme" type="text-short" column-name="request_scheme"/>
<field name="requestHost" type="text-short" column-name="request_host"/>
<field name="requestPath" type="text-medium" column-name="request_path"/>
<field name="requestQuery" type="text-long" column-name="request_query"/>
<field name="httpVersion" type="text-short" column-name="http_version"/>
<field name="responseCode" type="number-integer" column-name="response_code"/>
<field name="timeInitialMs" type="number-integer" column-name="time_initial_ms"/>
<field name="timeFinalMs" type="number-integer" column-name="time_final_ms"/>
<field name="bytesSent" type="number-integer" column-name="bytes_sent"/>
<field name="referrer" type="text-medium"/>
<field name="agent" type="text-medium"/>
<field name="sessionId" type="id" column-name="session_id"/>
<field name="visitorId" type="id" column-name="visitor_id"/>
<index name="MQHTTPLOG_TS" unique="false"><index-field name="logTimestamp"/></index>
<index name="MQHTTPLOG_PATH" unique="false"><index-field name="requestPath"/></index>
</entity>

</entities>
54 changes: 37 additions & 17 deletions framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import org.moqui.impl.entity.EntityDefinition
import org.moqui.impl.entity.EntityJavaUtil
import org.moqui.impl.entity.FieldInfo
import org.moqui.impl.util.ElasticSearchLogger
import org.moqui.impl.util.PostgresSearchLogger
import org.moqui.util.LiteStringMap
import org.moqui.util.MNode
import org.moqui.util.RestClient
Expand Down Expand Up @@ -69,8 +70,9 @@ class ElasticFacadeImpl implements ElasticFacade {
}

public final ExecutionContextFactoryImpl ecfi
private final Map<String, ElasticClientImpl> clientByClusterName = new LinkedHashMap<>()
private final Map<String, ElasticClient> clientByClusterName = new LinkedHashMap<>()
private ElasticSearchLogger esLogger = null
private PostgresSearchLogger pgLogger = null

ElasticFacadeImpl(ExecutionContextFactoryImpl ecfi) {
this.ecfi = ecfi
Expand All @@ -90,14 +92,21 @@ class ElasticFacadeImpl implements ElasticFacade {
logger.warn("ElasticFacade Client for cluster ${clusterName} already initialized, skipping")
continue
}
if (!clusterUrl) {
logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping")
continue
}

String clusterType = clusterNode.attribute("type") ?: "elastic"
try {
ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi)
clientByClusterName.put(clusterName, elci)
if ("postgres".equals(clusterType)) {
PostgresElasticClient pgc = new PostgresElasticClient(clusterNode, ecfi)
clientByClusterName.put(clusterName, pgc)
logger.info("Initialized PostgresElasticClient for cluster ${clusterName}")
} else {
if (!clusterUrl) {
logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping")
continue
}
ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi)
clientByClusterName.put(clusterName, elci)
}
} catch (Throwable t) {
Throwable cause = t.getCause()
if (cause != null && cause.message.contains("refused")) {
Expand All @@ -108,22 +117,29 @@ class ElasticFacadeImpl implements ElasticFacade {
}
}

// init ElasticSearchLogger
if (esLogger == null || !esLogger.isInitialized()) {
ElasticClientImpl loggerEci = clientByClusterName.get("logger") ?: clientByClusterName.get("default")
if (loggerEci != null) {
logger.info("Initializing ElasticSearchLogger with cluster ${loggerEci.getClusterName()}")
esLogger = new ElasticSearchLogger(loggerEci, ecfi)
// init ElasticSearchLogger / PostgresSearchLogger depending on backend type
ElasticClient loggerClient = clientByClusterName.get("logger") ?: clientByClusterName.get("default")
if (loggerClient instanceof PostgresElasticClient) {
if (pgLogger == null || !pgLogger.isInitialized()) {
logger.info("Initializing PostgresSearchLogger with cluster ${loggerClient.getClusterName()}")
pgLogger = new PostgresSearchLogger((PostgresElasticClient) loggerClient, ecfi)
} else {
logger.warn("PostgresSearchLogger in place and initialized, skipping")
}
} else if (loggerClient instanceof ElasticClientImpl) {
if (esLogger == null || !esLogger.isInitialized()) {
logger.info("Initializing ElasticSearchLogger with cluster ${loggerClient.getClusterName()}")
esLogger = new ElasticSearchLogger((ElasticClientImpl) loggerClient, ecfi)
} else {
logger.warn("No Elastic Client found with name 'logger' or 'default', not initializing ElasticSearchLogger")
logger.warn("ElasticSearchLogger in place and initialized, skipping")
}
} else {
logger.warn("ElasticSearchLogger in place and initialized, not initializing ElasticSearchLogger")
logger.warn("No Elastic/Postgres Client found with name 'logger' or 'default', not initializing search logger")
}

// Index DataFeed with indexOnStartEmpty=Y
try {
ElasticClientImpl defaultEci = clientByClusterName.get("default")
ElasticClient defaultEci = clientByClusterName.get("default")
if (defaultEci != null) {
EntityList dataFeedList = ecfi.entityFacade.find("moqui.entity.feed.DataFeed")
.condition("indexOnStartEmpty", "Y").disableAuthz().list()
Expand Down Expand Up @@ -151,7 +167,11 @@ class ElasticFacadeImpl implements ElasticFacade {

void destroy() {
if (esLogger != null) esLogger.destroy()
for (ElasticClientImpl eci in clientByClusterName.values()) eci.destroy()
if (pgLogger != null) pgLogger.destroy()
for (ElasticClient eci in clientByClusterName.values()) {
if (eci instanceof ElasticClientImpl) ((ElasticClientImpl) eci).destroy()
else if (eci instanceof PostgresElasticClient) ((PostgresElasticClient) eci).destroy()
}
}

@Override ElasticClient getDefault() { return clientByClusterName.get("default") }
Expand Down
Loading