Conformance tests

.github/workflows/release.yml

@@ -20,17 +20,8 @@ jobs:
     env:
       PG_CONFIG: /usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config
 
-
     steps:
     - uses: actions/checkout@v3
-      with:
-        submodules: true
-        lfs: false
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y build-essential libprotobuf-c-dev protobuf-c-compiler
 
     - name: Setup PostgreSQL ${{ matrix.pg_version }}
       run: |
@@ -64,8 +55,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-      with:
-        submodules: recursive
 
     - name: Set Version
       run: |
@@ -75,18 +64,13 @@ jobs:
     - name: Package Source Bundle for PGXN
       run: |
         mkdir -p dist/pgproto-${{ env.VERSION }}
-        # Copy source files
         cp -R src/ dist/pgproto-${{ env.VERSION }}/
         cp -R sql/ dist/pgproto-${{ env.VERSION }}/
         cp Makefile dist/pgproto-${{ env.VERSION }}/
         cp pgproto.control dist/pgproto-${{ env.VERSION }}/
         cp META.json dist/pgproto-${{ env.VERSION }}/
         cp LICENSE dist/pgproto-${{ env.VERSION }}/
         cp README.md dist/pgproto-${{ env.VERSION }}/
-        cp -R third_party/ dist/pgproto-${{ env.VERSION }}/
-        # Debug: check if files are there
-        ls -la dist/pgproto-${{ env.VERSION }}/third_party/protobuf/upb/reflection/def_pool.h || echo "FILE NOT FOUND"
-        # Zip it
         cd dist && zip -r ../pgproto-${{ env.VERSION }}.zip pgproto-${{ env.VERSION }}/
 
     - name: Upload Source Release Asset
@@ -122,10 +106,8 @@ jobs:
         sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
         wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
         sudo apt-get update
-        sudo apt-get install -y postgresql-server-dev-18 libprotobuf-c-dev protobuf-c-compiler python3-pip
+        sudo apt-get install -y postgresql-server-dev-18 python3-pip
         sudo pip3 install pgxnclient
-    - name: List files
-      run: ls -la
     - name: Test PGXN Install
       run: |
         sudo pgxn install ./pgproto-${{ env.VERSION }}.zip
@@ -135,8 +117,6 @@ jobs:
     needs: build-binaries
     steps:
     - uses: actions/checkout@v3
-      with:
-        submodules: true
     - name: Log in to GHCR
       uses: docker/login-action@v2
       with:
@@ -161,8 +141,6 @@ jobs:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v3
-      with:
-        submodules: true
     - name: Setup MSYS2
       uses: msys2/setup-msys2@v2
       with:
@@ -171,7 +149,6 @@ jobs:
           make
           zip
           mingw-w64-x86_64-gcc
-          mingw-w64-x86_64-protobuf-c
           mingw-w64-x86_64-postgresql
     - name: Build
       run: |

Makefile

@@ -4,7 +4,7 @@ PG_CPPFLAGS = -std=c99 -I.
 
 EXTENSION = pgproto
 DATA = sql/pgproto--1.0.sql
-REGRESS = pgproto_test
+REGRESS = pgproto_test conformance_test
 
 PG_CONFIG ?= pg_config
 PGXS := $(shell "$(PG_CONFIG)" --pgxs)

expected/conformance_test.out

@@ -0,0 +1,139 @@
+-- Conformance Test Suite for pgproto
+CREATE EXTENSION IF NOT EXISTS pgproto;
+NOTICE:  extension "pgproto" already exists, skipping
+CREATE TABLE pb_conformance (id serial, data protobuf);
+-- Register conformance schema
+INSERT INTO pb_schemas (name, data) VALUES ('conformance.proto', decode('0a8d080a1f746573745f646174615f6e65772f6c6f6e666f726d616e63652e70726f746f120b636f6e666f726d616e636522d4070a0e436f6e666f726d616e63654d736712190a08665f646f75626c65180120012801520766446f75626c6512170a07665f666c6f6174180220012802520666466c6f617412170a07665f696e743634180320012803520666496e74363412190a08665f75696e74363418042001280452076655696e74363412170a07665f696e743332180520012805520666496e743332121b0a09665f6669786564363418062001280652086646697865643634121b0a09665f666978656433321807200128075208664669786564333212150a06665f626f6f6c180820012808520566426f6f6c12190a08665f737472696e67180920012809520766537472696e6712170a07665f6279746573180a2001280c520666427974657312190a08665f75696e743332180b2001280d52076655696e743332121d0a0a665f7366697865643332180c2001280f5209665366697865643332121d0a0a665f7366697865643634180d20012810520966536669786564363412190a08665f73696e743332180e2001281152076653696e74333212190a08665f73696e743634180f2001281252076653696e74363412170a07725f696e743332181020032805520672496e74333212280a0e725f696e7433325f7061636b656418112003280542021001520c72496e7433325061636b656412190a08725f737472696e67181220032809520772537472696e6712440a096d5f7374725f73747218132003280b32282e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4d537472537472456e74727952076d53747253747212440a096d5f696e745f696e7418142003280b32282e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4d496e74496e74456e74727952076d496e74496e7412150a05635f696e741815200128054800520463496e7412150a05635f7374721816200128094800520463537472123d0a08665f6e657374656418172001280b32222e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4e65737465645207664e65737465641a3a0a0c4d537472537472456e74727912100a036b657918012001280952036b657912140a0576616c7565180220012809520576616c75653a0238011a3a0a0c4d496e74496e74456e74727912100a036b657918012001280552036b657912140a0576616c7565180220012805520576616c75653a0238011a540a064e657374656412100a0376616c180120012805520376616c12380a05696e6e657218022001280b32222e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4e65737465645205696e6e657242080a0663686f696365620670726f746f33', 'hex'));
+-- 1. All Scalars Test
+-- Payload: 09ae47e17a14aef33f1585eb914018d285d8cc0420eaadc0e524282a314e61bc00000000003db17f390540014a0b68656c6c6f20776f726c64520b62696e617279206461746158646585ffffff6938feffffffffffff70a90c78e50f
+INSERT INTO pb_conformance (data) VALUES (decode('09ae47e17a14aef33f1585eb914018d285d8cc0420eaadc0e524282a314e61bc00000000003db17f390540014a0b68656c6c6f20776f726c64520b62696e617279206461746158646585ffffff6938feffffffffffff70a90c78e50f', 'hex'));
+ERROR:  column "data" is of type protobuf but expression is of type bytea
+LINE 1: INSERT INTO pb_conformance (data) VALUES (decode('09ae47e17a...
+                                                  ^
+HINT:  You will need to rewrite or cast the expression.
+SELECT pb_get_int32(data, 5) AS f_int32 FROM pb_conformance WHERE id = 1;
+ f_int32 
+---------
+(0 rows)
+
+SELECT pb_get_int32(data, 11) AS f_uint32 FROM pb_conformance WHERE id = 1;
+ f_uint32 
+----------
+(0 rows)
+
+SELECT pb_get_int32(data, 8) AS f_bool FROM pb_conformance WHERE id = 1;
+ f_bool 
+--------
+(0 rows)
+
+SELECT data -> 'conformance.ConformanceMsg.f_string'::text AS f_string FROM pb_conformance WHERE id = 1;
+ f_string 
+----------
+(0 rows)
+
+-- 2. Repeated Packed Test
+-- Payload: 8a01030a141e (Tag 17, Length 3, Values 10, 20, 30)
+INSERT INTO pb_conformance (data) VALUES (decode('8a01030a141e', 'hex'));
+ERROR:  column "data" is of type protobuf but expression is of type bytea
+LINE 1: INSERT INTO pb_conformance (data) VALUES (decode('8a01030a14...
+                                                  ^
+HINT:  You will need to rewrite or cast the expression.
+-- Path access to packed repeated
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 0}'::text[] FROM pb_conformance WHERE id = 2;
+ ?column? 
+----------
+(0 rows)
+
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 1}'::text[] FROM pb_conformance WHERE id = 2;
+ ?column? 
+----------
+(0 rows)
+
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 2}'::text[] FROM pb_conformance WHERE id = 2;
+ ?column? 
+----------
+(0 rows)
+
+-- 3. Maps Test
+-- Payload: 9a01080a026b31120276319a01080a026b3212027632a2010408011064
+INSERT INTO pb_conformance (data) VALUES (decode('9a01080a026b31120276319a01080a026b3212027632a2010408011064', 'hex'));
+ERROR:  column "data" is of type protobuf but expression is of type bytea
+LINE 1: INSERT INTO pb_conformance (data) VALUES (decode('9a01080a02...
+                                                  ^
+HINT:  You will need to rewrite or cast the expression.
+SELECT data #> '{conformance.ConformanceMsg, m_str_str, k1}'::text[] FROM pb_conformance WHERE id = 3;
+ ?column? 
+----------
+(0 rows)
+
+SELECT data #> '{conformance.ConformanceMsg, m_int_int, 1}'::text[] FROM pb_conformance WHERE id = 3;
+ ?column? 
+----------
+(0 rows)
+
+-- 4. Oneof Test
+-- Payload: b2010b6f6e656f665f76616c7565 (Tag 22, "oneof_value")
+INSERT INTO pb_conformance (data) VALUES (decode('b2010b6f6e656f665f76616c7565', 'hex'));
+ERROR:  column "data" is of type protobuf but expression is of type bytea
+LINE 1: INSERT INTO pb_conformance (data) VALUES (decode('b2010b6f6e...
+                                                  ^
+HINT:  You will need to rewrite or cast the expression.
+SELECT data #> '{conformance.ConformanceMsg, c_str}'::text[] FROM pb_conformance WHERE id = 4;
+ ?column? 
+----------
+(0 rows)
+
+SELECT data #> '{conformance.ConformanceMsg, c_int}'::text[] FROM pb_conformance WHERE id = 4; -- Should be NULL
+ ?column? 
+----------
+(0 rows)
+
+-- 5. Nested Message Test
+-- Payload: ba0107087b120308c803 (Tag 23, Nested { val: 123, inner { val: 456 } })
+INSERT INTO pb_conformance (data) VALUES (decode('ba0107087b120308c803', 'hex'));
+ERROR:  column "data" is of type protobuf but expression is of type bytea
+LINE 1: INSERT INTO pb_conformance (data) VALUES (decode('ba0107087b...
+                                                  ^
+HINT:  You will need to rewrite or cast the expression.
+SELECT data #> '{conformance.ConformanceMsg, f_nested, val}'::text[] FROM pb_conformance WHERE id = 5;
+ ?column? 
+----------
+(0 rows)
+
+SELECT data #> '{conformance.ConformanceMsg, f_nested, inner, val}'::text[] FROM pb_conformance WHERE id = 5;
+ ?column? 
+----------
+(0 rows)
+
+-- 6. Compaction & Bloat Verification
+-- Original size of id=1
+SELECT length(data::bytea) AS original_size FROM pb_conformance WHERE id = 1;
+ERROR:  cannot cast type protobuf to bytea
+LINE 1: SELECT length(data::bytea) AS original_size FROM pb_conforma...
+                          ^
+-- Multiple updates to same field
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '100') WHERE id = 1;
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '200') WHERE id = 1;
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '300') WHERE id = 1;
+-- Verify size remains constant (or even shrinks if 300 takes same space as 42)
+SELECT length(data::bytea) AS final_size FROM pb_conformance WHERE id = 1;
+ERROR:  cannot cast type protobuf to bytea
+LINE 1: SELECT length(data::bytea) AS final_size FROM pb_conformance...
+                          ^
+-- Verify value
+SELECT pb_get_int32(data, 5) FROM pb_conformance WHERE id = 1;
+ pb_get_int32 
+--------------
+(0 rows)
+
+-- 7. Deletion & Compaction
+UPDATE pb_conformance SET data = pb_delete(data, ARRAY['conformance.ConformanceMsg', 'f_string']) WHERE id = 1;
+SELECT length(data::bytea) AS size_after_delete FROM pb_conformance WHERE id = 1;
+ERROR:  cannot cast type protobuf to bytea
+LINE 1: SELECT length(data::bytea) AS size_after_delete FROM pb_conf...
+                          ^
+SELECT data -> 'conformance.ConformanceMsg.f_string'::text FROM pb_conformance WHERE id = 1;
+ ?column? 
+----------
+(0 rows)
+

expected/pgproto_test.out

@@ -420,7 +420,7 @@ SELECT pb_insert('\x'::protobuf, ARRAY['Outer', 'nonexistent', '0'], '100');
 ERROR:  Field nonexistent not found in message Outer
 -- 7. Test pb_insert error: Not a repeated or map field
 SELECT pb_insert('\x'::protobuf, ARRAY['Outer', 'inner', '0'], '100');
-ERROR:  Field not repeated or map
+ERROR:  Field inner is not a repeated or map field
 -- 8. Test pb_delete for array field 'scores' in Outer
 SELECT pb_to_json(pb_delete(pb_insert(pb_insert('\x'::protobuf, ARRAY['Outer', 'scores', '0'], '100'), ARRAY['Outer', 'scores', '1'], '200'), ARRAY['Outer', 'scores', '0']), 'Outer');
  pb_to_json 

sql/conformance_test.sql

@@ -0,0 +1,58 @@
+-- Conformance Test Suite for pgproto
+CREATE EXTENSION IF NOT EXISTS pgproto;
+CREATE TABLE pb_conformance (id serial, data protobuf);
+
+-- Register conformance schema
+INSERT INTO pb_schemas (name, data) VALUES ('conformance.proto', decode('0a8d080a1f746573745f646174615f6e65772f6c6f6e666f726d616e63652e70726f746f120b636f6e666f726d616e636522d4070a0e436f6e666f726d616e63654d736712190a08665f646f75626c65180120012801520766446f75626c6512170a07665f666c6f6174180220012802520666466c6f617412170a07665f696e743634180320012803520666496e74363412190a08665f75696e74363418042001280452076655696e74363412170a07665f696e743332180520012805520666496e743332121b0a09665f6669786564363418062001280652086646697865643634121b0a09665f666978656433321807200128075208664669786564333212150a06665f626f6f6c180820012808520566426f6f6c12190a08665f737472696e67180920012809520766537472696e6712170a07665f6279746573180a2001280c520666427974657312190a08665f75696e743332180b2001280d52076655696e743332121d0a0a665f7366697865643332180c2001280f5209665366697865643332121d0a0a665f7366697865643634180d20012810520966536669786564363412190a08665f73696e743332180e2001281152076653696e74333212190a08665f73696e743634180f2001281252076653696e74363412170a07725f696e743332181020032805520672496e74333212280a0e725f696e7433325f7061636b656418112003280542021001520c72496e7433325061636b656412190a08725f737472696e67181220032809520772537472696e6712440a096d5f7374725f73747218132003280b32282e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4d537472537472456e74727952076d53747253747212440a096d5f696e745f696e7418142003280b32282e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4d496e74496e74456e74727952076d496e74496e7412150a05635f696e741815200128054800520463496e7412150a05635f7374721816200128094800520463537472123d0a08665f6e657374656418172001280b32222e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4e65737465645207664e65737465641a3a0a0c4d537472537472456e74727912100a036b657918012001280952036b657912140a0576616c7565180220012809520576616c75653a0238011a3a0a0c4d496e74496e74456e74727912100a036b657918012001280552036b657912140a0576616c7565180220012805520576616c75653a0238011a540a064e657374656412100a0376616c180120012805520376616c12380a05696e6e657218022001280b32222e636f6e666f726d616e63652e436f6e666f726d616e63654d73672e4e65737465645205696e6e657242080a0663686f696365620670726f746f33', 'hex'));
+
+-- 1. All Scalars Test
+-- Payload: 09ae47e17a14aef33f1585eb914018d285d8cc0420eaadc0e524282a314e61bc00000000003db17f390540014a0b68656c6c6f20776f726c64520b62696e617279206461746158646585ffffff6938feffffffffffff70a90c78e50f
+INSERT INTO pb_conformance (data) VALUES (decode('09ae47e17a14aef33f1585eb914018d285d8cc0420eaadc0e524282a314e61bc00000000003db17f390540014a0b68656c6c6f20776f726c64520b62696e617279206461746158646585ffffff6938feffffffffffff70a90c78e50f', 'hex'));
+
+SELECT pb_get_int32(data, 5) AS f_int32 FROM pb_conformance WHERE id = 1;
+SELECT pb_get_int32(data, 11) AS f_uint32 FROM pb_conformance WHERE id = 1;
+SELECT pb_get_int32(data, 8) AS f_bool FROM pb_conformance WHERE id = 1;
+SELECT data -> 'conformance.ConformanceMsg.f_string'::text AS f_string FROM pb_conformance WHERE id = 1;
+
+-- 2. Repeated Packed Test
+-- Payload: 8a01030a141e (Tag 17, Length 3, Values 10, 20, 30)
+INSERT INTO pb_conformance (data) VALUES (decode('8a01030a141e', 'hex'));
+-- Path access to packed repeated
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 0}'::text[] FROM pb_conformance WHERE id = 2;
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 1}'::text[] FROM pb_conformance WHERE id = 2;
+SELECT data #> '{conformance.ConformanceMsg, r_int32_packed, 2}'::text[] FROM pb_conformance WHERE id = 2;
+
+-- 3. Maps Test
+-- Payload: 9a01080a026b31120276319a01080a026b3212027632a2010408011064
+INSERT INTO pb_conformance (data) VALUES (decode('9a01080a026b31120276319a01080a026b3212027632a2010408011064', 'hex'));
+SELECT data #> '{conformance.ConformanceMsg, m_str_str, k1}'::text[] FROM pb_conformance WHERE id = 3;
+SELECT data #> '{conformance.ConformanceMsg, m_int_int, 1}'::text[] FROM pb_conformance WHERE id = 3;
+
+-- 4. Oneof Test
+-- Payload: b2010b6f6e656f665f76616c7565 (Tag 22, "oneof_value")
+INSERT INTO pb_conformance (data) VALUES (decode('b2010b6f6e656f665f76616c7565', 'hex'));
+SELECT data #> '{conformance.ConformanceMsg, c_str}'::text[] FROM pb_conformance WHERE id = 4;
+SELECT data #> '{conformance.ConformanceMsg, c_int}'::text[] FROM pb_conformance WHERE id = 4; -- Should be NULL
+
+-- 5. Nested Message Test
+-- Payload: ba0107087b120308c803 (Tag 23, Nested { val: 123, inner { val: 456 } })
+INSERT INTO pb_conformance (data) VALUES (decode('ba0107087b120308c803', 'hex'));
+SELECT data #> '{conformance.ConformanceMsg, f_nested, val}'::text[] FROM pb_conformance WHERE id = 5;
+SELECT data #> '{conformance.ConformanceMsg, f_nested, inner, val}'::text[] FROM pb_conformance WHERE id = 5;
+
+-- 6. Compaction & Bloat Verification
+-- Original size of id=1
+SELECT length(data::bytea) AS original_size FROM pb_conformance WHERE id = 1;
+-- Multiple updates to same field
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '100') WHERE id = 1;
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '200') WHERE id = 1;
+UPDATE pb_conformance SET data = pb_set(data, ARRAY['conformance.ConformanceMsg', 'f_int32'], '300') WHERE id = 1;
+-- Verify size remains constant (or even shrinks if 300 takes same space as 42)
+SELECT length(data::bytea) AS final_size FROM pb_conformance WHERE id = 1;
+-- Verify value
+SELECT pb_get_int32(data, 5) FROM pb_conformance WHERE id = 1;
+
+-- 7. Deletion & Compaction
+UPDATE pb_conformance SET data = pb_delete(data, ARRAY['conformance.ConformanceMsg', 'f_string']) WHERE id = 1;
+SELECT length(data::bytea) AS size_after_delete FROM pb_conformance WHERE id = 1;
+SELECT data -> 'conformance.ConformanceMsg.f_string'::text FROM pb_conformance WHERE id = 1;