From f5f1ab06c31d2231e10d2f317058d9557250fd6a Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Fri, 14 Nov 2025 15:26:36 +0100 Subject: [PATCH 1/6] delete-tablewithoutindex --- dblinter/default_config.yaml | 8 -------- dblinter/rules/T002/TableWithoutIndex.py | 26 ------------------------ 2 files changed, 34 deletions(-) delete mode 100644 dblinter/rules/T002/TableWithoutIndex.py diff --git a/dblinter/default_config.yaml b/dblinter/default_config.yaml index f32c301..7699af3 100644 --- a/dblinter/default_config.yaml +++ b/dblinter/default_config.yaml @@ -103,14 +103,6 @@ table: message: "No primary key on table {0}.{1}.{2}." fixes: - create a primary key. - - name: TableWithoutIndex - ruleid: T002 - enabled: True - context: - desc: table without index. - message: "No index on table {0}.{1}.{2}." - fixes: - - check if it's necessary to create index. - name: TableWithRedundantIndex ruleid: T003 enabled: True diff --git a/dblinter/rules/T002/TableWithoutIndex.py b/dblinter/rules/T002/TableWithoutIndex.py deleted file mode 100644 index ac0c7bc..0000000 --- a/dblinter/rules/T002/TableWithoutIndex.py +++ /dev/null @@ -1,26 +0,0 @@ -import logging - -from dblinter.database_connection import DatabaseConnection -from dblinter.function_library import EXCLUDED_SCHEMAS_STR - -LOGGER = logging.getLogger("dblinter") - - -def table_without_index( - self, db: DatabaseConnection, _, context, table, sarif_document -): - LOGGER.debug( - "table_without_index for %s.%s in db %s", table[0], table[1], db.database - ) - NB_TABLE_WITHOUT_INDEX = """SELECT count(*) FROM pg_catalog.pg_indexes WHERE schemaname='{}' AND tablename='{}' - AND schemaname NOT IN ('{}')""" - - uri = f"{db.database}.{table[0]}.{table[1]}" - index_count = db.query( - NB_TABLE_WITHOUT_INDEX.format(table[0], table[1], EXCLUDED_SCHEMAS_STR) - )[0][0] - if index_count == 0: - message_args = (db.database, table[0], table[1]) - sarif_document.add_check( - self.get_ruleid_from_function_name(), message_args, uri, context - ) From 35e0eb1af1e3068e0f7c55d4c31da70b5f52000f Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Fri, 14 Nov 2025 16:38:52 +0100 Subject: [PATCH 2/6] delete related tests --- dblinter/doc/TABLE002-TableWithoutIndex.md | 0 tests/data/good_config.yaml | 8 ---- tests/rules/T002/test_TableWithoutIndex.py | 46 ---------------------- 3 files changed, 54 deletions(-) delete mode 100644 dblinter/doc/TABLE002-TableWithoutIndex.md delete mode 100644 tests/rules/T002/test_TableWithoutIndex.py diff --git a/dblinter/doc/TABLE002-TableWithoutIndex.md b/dblinter/doc/TABLE002-TableWithoutIndex.md deleted file mode 100644 index e69de29..0000000 diff --git a/tests/data/good_config.yaml b/tests/data/good_config.yaml index 2205a44..4e6c5fe 100644 --- a/tests/data/good_config.yaml +++ b/tests/data/good_config.yaml @@ -72,14 +72,6 @@ table: message: "No primary key on table {0}.{1}.{2}." fixes: - create a primary key. - - name: TableWithoutIndex - ruleid: T002 - enabled: True - context: - desc: table without index. - message: "No index on table {0}.{1}.{2}." - fixes: - - check if it's necessary to create index. - name: TableWithRedundantIndex ruleid: T003 enabled: True diff --git a/tests/rules/T002/test_TableWithoutIndex.py b/tests/rules/T002/test_TableWithoutIndex.py deleted file mode 100644 index 7941c94..0000000 --- a/tests/rules/T002/test_TableWithoutIndex.py +++ /dev/null @@ -1,46 +0,0 @@ -from dblinter.configuration_model import Context -from dblinter.database_connection import DatabaseConnection -from dblinter.function_library import FunctionLibrary -from dblinter.sarif_document import SarifDocument - - -def test_table_without_index(postgres_instance_args) -> None: - args = postgres_instance_args - context = Context( - desc="Count number of tables without index on foreign key.", - message="No index on table {0}.{1}.{2}", - fixes=[ - "create a index on foreign key or change warning/error threshold.", - ], - ) - db = DatabaseConnection(args) - sarif_document = SarifDocument() - function_library = FunctionLibrary() - db.query("CREATE TABLE e_table1 (id integer, field1 text)") - function_library.get_function_by_function_name("table_without_index")( - function_library, db, [], context, ("public", "e_table1"), sarif_document - ) - assert ( - sarif_document.sarif_doc.runs[0].results[0].message.text - == "No index on table postgres.public.e_table1" - ) - - -def test_table_with_index(postgres_instance_args) -> None: - args = postgres_instance_args - context = Context( - desc="Count number of tables without index on foreign key.", - message="No index on table {0}.{1}.{2}.", - fixes=[ - "create a index on foreign key or change warning/error threshold.", - ], - ) - db = DatabaseConnection(args) - sarif_document = SarifDocument() - function_library = FunctionLibrary() - db.query("CREATE TABLE f_table1 (id integer, field1 text)") - db.query("CREATE index on f_table1(id)") - function_library.get_function_by_function_name("table_without_index")( - function_library, db, [], context, ("public", "f_table1"), sarif_document - ) - assert sarif_document.sarif_doc.runs[0].results == [] From e4968269cbe2af5562934a46161e3e02cbdc2316 Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Mon, 17 Nov 2025 16:17:35 +0100 Subject: [PATCH 3/6] fix config and add cleanup fixture --- dblinter.cfg | 8 ----- tests/conftest.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 36 ++++---------------- 3 files changed, 90 insertions(+), 38 deletions(-) diff --git a/dblinter.cfg b/dblinter.cfg index 9df281d..8596477 100644 --- a/dblinter.cfg +++ b/dblinter.cfg @@ -72,14 +72,6 @@ table: message: "No primary key on table {0}.{1}.{2}." fixes: - create a primary key. - - name: TableWithoutIndex - ruleid: T002 - enabled: True - context: - desc: table without index. - message: "No index on table {0}.{1}.{2}." - fixes: - - check if it's necessary to create index. - name: TableWithRedundantIndex ruleid: T003 enabled: True diff --git a/tests/conftest.py b/tests/conftest.py index 532f7ee..8dcb66b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,3 +81,87 @@ def remove_container(): } wait_for_postgres(uri, 30) return uri + + +@pytest.fixture(scope="function", autouse=True) +def cleanup_database(request): + """Clean up database after each test to prevent test pollution. + + This fixture runs automatically for all tests that use postgres databases. + """ + # Only run cleanup if the test uses a postgres fixture + uses_pg = "postgres_instance_args" in request.fixturenames + uses_pg12 = "postgres12_instance_args" in request.fixturenames + + if not uses_pg and not uses_pg12: + yield + return + + yield + # Cleanup after test + + # Determine which instance to clean + instance_args = None + if uses_pg: + instance_args = request.getfixturevalue("postgres_instance_args") + elif uses_pg12: + instance_args = request.getfixturevalue("postgres12_instance_args") + + if not instance_args: + return + + try: + conn = psycopg2.connect( + user=instance_args["user"], + password=instance_args["password"], + host=instance_args["host"], + port=instance_args["port"], + dbname=instance_args["dbname"], + sslmode=instance_args["sslmode"], + connect_timeout=10, + ) + conn.autocommit = True + cur = conn.cursor() + + # Drop all user-created schemas (except public) + cur.execute(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public', + 'pg_toast', '_timescaledb_catalog', '_timescaledb_config', + '_timescaledb_internal', '_timescaledb_cache', '_timescaledb_functions', + 'timescaledb', 'pgaudit') + """) + schemas = cur.fetchall() + for (schema,) in schemas: + cur.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') + + # Drop all tables in public schema + cur.execute(""" + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + """) + tables = cur.fetchall() + for (table,) in tables: + cur.execute(f'DROP TABLE IF EXISTS public."{table}" CASCADE') + + # Drop all user-created roles + cur.execute(""" + SELECT rolname + FROM pg_roles + WHERE rolname NOT LIKE 'pg_%' + AND rolname != 'postgres' + """) + roles = cur.fetchall() + for (role,) in roles: + # Revoke all privileges first + cur.execute(f'REASSIGN OWNED BY "{role}" TO postgres') + cur.execute(f'DROP OWNED BY "{role}" CASCADE') + cur.execute(f'DROP ROLE IF EXISTS "{role}"') + + cur.close() + conn.close() + except Exception as e: + # If cleanup fails, log but don't fail the test + print(f"Cleanup warning: {e}") diff --git a/tests/test_main.py b/tests/test_main.py index 80221b8..290ab91 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -299,22 +299,10 @@ def test_main_with_include_table(postgres_instance_args) -> None: sarif_document=sarif_document, include="e_table%", ) - assert ( - sarif_document.sarif_doc.runs[0].results[0].message.text - == "No primary key on table postgres.schema1.e_table1." - ) - assert ( - sarif_document.sarif_doc.runs[0].results[1].message.text - == "No index on table postgres.schema1.e_table1." - ) - assert ( - sarif_document.sarif_doc.runs[0].results[3].message.text - == "No primary key on table postgres.schema1.e_table2." - ) - assert ( - sarif_document.sarif_doc.runs[0].results[4].message.text - == "No index on table postgres.schema1.e_table2." - ) + # Check that results contain expected messages (order not guaranteed) + result_messages = [r.message.text for r in sarif_document.sarif_doc.runs[0].results] + assert "No primary key on table postgres.schema1.e_table1." in result_messages + assert "No primary key on table postgres.schema1.e_table2." in result_messages def test_main_with_include_table_and_schema(postgres_instance_args) -> None: @@ -347,17 +335,9 @@ def test_main_with_include_table_and_schema(postgres_instance_args) -> None: == "No primary key on table postgres.schema1.e_table1." ) assert ( - sarif_document.sarif_doc.runs[0].results[1].message.text - == "No index on table postgres.schema1.e_table1." - ) - assert ( - sarif_document.sarif_doc.runs[0].results[3].message.text + sarif_document.sarif_doc.runs[0].results[2].message.text == "No primary key on table postgres.schema1.e_table2." ) - assert ( - sarif_document.sarif_doc.runs[0].results[4].message.text - == "No index on table postgres.schema1.e_table2." - ) def test_main_with_exclude_table(postgres_instance_args) -> None: @@ -387,10 +367,6 @@ def test_main_with_exclude_table(postgres_instance_args) -> None: sarif_document.sarif_doc.runs[0].results[0].message.text == "No primary key on table postgres.schema1.e_table1." ) - assert ( - sarif_document.sarif_doc.runs[0].results[1].message.text - == "No index on table postgres.schema1.e_table1." - ) def test_main_with_schema_and_role_without_default_role_nok( @@ -501,7 +477,7 @@ def test_main_with_schema_and_role_not_exist_nok(postgres_instance_args) -> None assert ( "No role grantee on table postgres.schema1.e_table1. It means that except owner." - in sarif_document.sarif_doc.runs[0].results[2].message.text + in sarif_document.sarif_doc.runs[0].results[1].message.text ) From 051735714fc914930c504b3b634366fb49e93907 Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Mon, 17 Nov 2025 16:27:16 +0100 Subject: [PATCH 4/6] fix black formatter --- .github/workflows/build-main.yaml | 2 +- .github/workflows/build-pr.yaml | 2 +- pyproject.toml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 7da5d81..2405c12 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -28,7 +28,7 @@ jobs: sudo apt install libpq-dev pip install poetry - run: poetry install --with dev - - run: poetry run isort --profile black --diff . + - run: poetry run isort --profile black --diff dblinter/ tests/ - run: poetry run black --check . - run: poetry run ruff check . - run: poetry run pylint dblinter tests diff --git a/.github/workflows/build-pr.yaml b/.github/workflows/build-pr.yaml index 03aee84..0fc3287 100644 --- a/.github/workflows/build-pr.yaml +++ b/.github/workflows/build-pr.yaml @@ -26,7 +26,7 @@ jobs: sudo apt install libpq-dev pip install poetry - run: poetry install --with dev - - run: poetry run isort --profile black --diff . + - run: poetry run isort --profile black --diff dblinter/ tests/ - run: poetry run black --check . - run: poetry run ruff check . - run: poetry run pylint dblinter tests diff --git a/pyproject.toml b/pyproject.toml index 15b7fed..c3f3f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,5 @@ extension-pkg-whitelist = "pydantic" requires = ["poetry-core"] build-backend = "poetry-core.masonry.api" +[tool.black] +extend-exclude = "dblinter.cfg" From c3096034596ca9ec026a2248264c8cda5ee46f19 Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Tue, 18 Nov 2025 11:59:43 +0100 Subject: [PATCH 5/6] fix black checks --- tests/conftest.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8dcb66b..b8074ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -124,35 +124,41 @@ def cleanup_database(request): cur = conn.cursor() # Drop all user-created schemas (except public) - cur.execute(""" + cur.execute( + """ SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public', 'pg_toast', '_timescaledb_catalog', '_timescaledb_config', '_timescaledb_internal', '_timescaledb_cache', '_timescaledb_functions', 'timescaledb', 'pgaudit') - """) + """ + ) schemas = cur.fetchall() for (schema,) in schemas: cur.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') # Drop all tables in public schema - cur.execute(""" + cur.execute( + """ SELECT tablename FROM pg_tables WHERE schemaname = 'public' - """) + """ + ) tables = cur.fetchall() for (table,) in tables: cur.execute(f'DROP TABLE IF EXISTS public."{table}" CASCADE') # Drop all user-created roles - cur.execute(""" + cur.execute( + """ SELECT rolname FROM pg_roles WHERE rolname NOT LIKE 'pg_%' AND rolname != 'postgres' - """) + """ + ) roles = cur.fetchall() for (role,) in roles: # Revoke all privileges first From 15155281cf074cf2b1b7bdcd3fcd7e08c62adf40 Mon Sep 17 00:00:00 2001 From: Eric VERLEENE Date: Tue, 18 Nov 2025 12:08:46 +0100 Subject: [PATCH 6/6] fix exception --- tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b8074ec..5089718 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -168,6 +168,5 @@ def cleanup_database(request): cur.close() conn.close() - except Exception as e: - # If cleanup fails, log but don't fail the test + except Exception as e: # pylint: disable=broad-exception-caught print(f"Cleanup warning: {e}")