diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc1123c1..4b533be9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ name: test types: [test] env: - KCIDB_IO_URL: kcidb-io@git+https://github.com/kernelci/kcidb-io.git@${{ github.event.client_payload.kcidb_io_ref || 'main' }} + KCIDB_IO_URL: kcidb-io@git+https://github.com/kernelci/kcidb-io.git@${{ github.event.client_payload.kcidb_io_ref || 'add_schema_v4_4' }} jobs: check_python: diff --git a/.pre-commit b/.pre-commit index 45973b67..4e021540 100755 --- a/.pre-commit +++ b/.pre-commit @@ -2,7 +2,7 @@ set -e -u -o pipefail -if [ -n "$NO_VERIFY" ]; then +if [ -n "${NO_VERIFY:-}" ]; then echo "Skipping pre-commit hook" >&2 exit 0 fi diff --git a/kcidb/db/bigquery/__init__.py b/kcidb/db/bigquery/__init__.py index 9fe8b291..2b70b3c0 100644 --- a/kcidb/db/bigquery/__init__.py +++ b/kcidb/db/bigquery/__init__.py @@ -2,7 +2,7 @@ import textwrap from kcidb.db.schematic import Driver as SchematicDriver -from kcidb.db.bigquery.v04_02 import Schema as LatestSchema +from kcidb.db.bigquery.v04_03 import Schema as LatestSchema class Driver(SchematicDriver): diff --git a/kcidb/db/bigquery/v04_00.py b/kcidb/db/bigquery/v04_00.py index 1229667a..b76c3a34 100644 --- a/kcidb/db/bigquery/v04_00.py +++ b/kcidb/db/bigquery/v04_00.py @@ -504,6 +504,7 @@ class Schema(AbstractSchema): checkout="SELECT\n" " id,\n" " git_commit_hash,\n" + " NULL AS git_commit_generation,\n" " patchset_hash,\n" " origin,\n" " git_repository_url,\n" @@ -584,6 +585,20 @@ class Schema(AbstractSchema): ' "" AS comment,\n' ' "" AS misc\n' 'FROM UNNEST([])', + transition='SELECT\n' + ' "" AS id,\n' + ' 0 AS version,\n' + ' "" AS origin,\n' + ' "" AS issue_id,\n' + ' 0 AS issue_version,\n' + ' FALSE AS appearance,\n' + ' "" AS revision_before_git_commit_hash,\n' + ' "" AS revision_before_patchset_hash,\n' + ' "" AS revision_after_git_commit_hash,\n' + ' "" AS revision_after_patchset_hash,\n' + ' "" AS comment,\n' + ' "" AS misc\n' + 'FROM UNNEST([])', ) @classmethod diff --git a/kcidb/db/bigquery/v04_02.py b/kcidb/db/bigquery/v04_02.py index f39c58e3..fea7b1e5 100644 --- a/kcidb/db/bigquery/v04_02.py +++ b/kcidb/db/bigquery/v04_02.py @@ -6,6 +6,7 @@ from google.cloud.bigquery.schema import SchemaField as Field from google.cloud.bigquery.table import Table import kcidb.io as io +from kcidb.misc import merge_dicts from .v04_01 import Schema as PreviousSchema # Module's logger @@ -36,7 +37,10 @@ class Schema(PreviousSchema): # A map of table names to the dictionary of fields and the names of their # aggregation function, if any (the default is "ANY_VALUE"). AGGS_MAP = { - name: {TIMESTAMP_FIELD.name: "MAX"} + name: merge_dicts( + PreviousSchema.AGGS_MAP.get(name, {}), + {TIMESTAMP_FIELD.name: "MAX"} + ) for name in TABLE_MAP } diff --git a/kcidb/db/bigquery/v04_03.py b/kcidb/db/bigquery/v04_03.py new file mode 100644 index 00000000..6de40f4e --- /dev/null +++ b/kcidb/db/bigquery/v04_03.py @@ -0,0 +1,196 @@ +"""Kernel CI report database - BigQuery schema v4.3""" + +import logging +from google.cloud.bigquery.schema import SchemaField as Field +from google.cloud.bigquery.table import Table +from kcidb.misc import merge_dicts +import kcidb.io as io +from .v04_02 import Schema as PreviousSchema, TIMESTAMP_FIELD + +# Module's logger +LOGGER = logging.getLogger(__name__) + +# Git commit generation number field +GIT_COMMIT_GENERATION_FIELD = Field( + "git_commit_generation", "INTEGER", + description="The commit generation number", +) + +# Revision ID fields +REVISION_ID_FIELDS = ( + Field( + "git_commit_hash", "STRING", + description="The full commit hash of the checked out base " + "source code", + ), + Field( + "patchset_hash", "STRING", + description="The patchset hash", + ), +) + + +class Schema(PreviousSchema): + """BigQuery database schema v4.3""" + + # The schema's version. + version = (4, 3) + # The I/O schema the database schema supports + io = io.schema.V4_4 + + # A map of table names to their BigQuery schemas + TABLE_MAP = merge_dicts(PreviousSchema.TABLE_MAP, dict( + checkouts=PreviousSchema.TABLE_MAP["checkouts"] + [ + GIT_COMMIT_GENERATION_FIELD, + ], + transitions=[ + TIMESTAMP_FIELD, + Field( + "id", "STRING", + description="Transition ID", + ), + Field( + "version", "INTEGER", + description="Transition version number", + ), + Field( + "origin", "STRING", + description="The name of the CI system which submitted " + "the transition", + ), + Field( + "issue_id", "STRING", + description="ID of the transitioning issue", + ), + Field( + "issue_version", "INTEGER", + description="Version number of the transitioning issue", + ), + Field( + "appearance", "BOOL", + description="True if this is an issue appearance, " + "false if disappearance.", + ), + Field( + "revision_before", "RECORD", fields=REVISION_ID_FIELDS, + description="ID of the last-known revision before the " + "transition" + ), + Field( + "revision_after", "RECORD", fields=REVISION_ID_FIELDS, + description="ID of the first-known revision after the " + "transition", + ), + Field( + "comment", "STRING", + description="A human-readable comment regarding the " + "transition", + ), + Field( + "misc", "STRING", + description="Miscellaneous extra data about the " + "transition in JSON format", + ), + ], + )) + + # A map of table names and their "primary key" fields + KEYS_MAP = dict( + **PreviousSchema.KEYS_MAP, + transitions=("id", "version",), + ) + + # A map of table names to the dictionary of fields and the names of their + # aggregation function, if any (the default is "ANY_VALUE"). + AGGS_MAP = merge_dicts( + PreviousSchema.AGGS_MAP, + dict(transitions={TIMESTAMP_FIELD.name: "MAX"}) + ) + + # Queries for each type of raw object-oriented data + OO_QUERIES = merge_dicts( + PreviousSchema.OO_QUERIES, + checkout="SELECT\n" + " id,\n" + " git_commit_hash,\n" + # Implement this column + " git_commit_generation,\n" + " patchset_hash,\n" + " origin,\n" + " git_repository_url,\n" + " git_repository_branch,\n" + " tree_name,\n" + " message_id,\n" + " start_time,\n" + " log_url,\n" + " log_excerpt,\n" + " comment,\n" + " valid,\n" + " misc\n" + "FROM checkouts", + # Implement transitions + transition="SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " revision_before.git_commit_hash AS " + "revision_before_git_commit_hash,\n" + " revision_before.patchset_hash AS " + "revision_before_patchset_hash,\n" + " revision_after.git_commit_hash AS " + "revision_after_git_commit_hash,\n" + " revision_after.patchset_hash AS " + "revision_after_patchset_hash,\n" + " comment,\n" + " misc\n" + "FROM (\n" + " SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " revision_before,\n" + " revision_after,\n" + " comment,\n" + " misc,\n" + " ROW_NUMBER() OVER (\n" + " PARTITION BY id\n" + " ORDER BY version DESC\n" + " ) AS precedence\n" + " FROM transitions\n" + ")\n" + "WHERE precedence = 1", + ) + + @classmethod + def _inherit(cls, conn): + """ + Inerit the database data from the previous schema version (if any). + + Args: + conn: Connection to the database to inherit. The database must + comply with the previous version of the schema. + """ + assert isinstance(conn, cls.Connection) + # Add the "git_commit_generation" field to _checkouts + conn.query_create(f""" + ALTER TABLE `_checkouts` + ADD COLUMN IF NOT EXISTS + `{GIT_COMMIT_GENERATION_FIELD.name}` + {GIT_COMMIT_GENERATION_FIELD.field_type} + OPTIONS(description={GIT_COMMIT_GENERATION_FIELD.description!r}) + """).result() + # Update the checkouts view + view_ref = conn.dataset_ref.table('checkouts') + view = Table(view_ref) + view.view_query = cls._format_view_query(conn, 'checkouts') + conn.client.update_table(view, ["view_query"]) + # Create new tables + for table_name in cls.TABLE_MAP: + if table_name not in PreviousSchema.TABLE_MAP: + cls._create_table(conn, table_name) diff --git a/kcidb/db/postgresql/__init__.py b/kcidb/db/postgresql/__init__.py index 6d393758..6df74568 100644 --- a/kcidb/db/postgresql/__init__.py +++ b/kcidb/db/postgresql/__init__.py @@ -2,7 +2,7 @@ import textwrap from kcidb.db.schematic import Driver as SchematicDriver -from kcidb.db.postgresql.v04_06 import Schema as LatestSchema +from kcidb.db.postgresql.v04_07 import Schema as LatestSchema class Driver(SchematicDriver): diff --git a/kcidb/db/postgresql/v04_00.py b/kcidb/db/postgresql/v04_00.py index 93b6f22b..faf47516 100644 --- a/kcidb/db/postgresql/v04_00.py +++ b/kcidb/db/postgresql/v04_00.py @@ -304,6 +304,7 @@ class Schema(AbstractSchema): statement="SELECT\n" " id,\n" " git_commit_hash,\n" + " NULL AS git_commit_generation,\n" " patchset_hash,\n" " origin,\n" " git_repository_url,\n" @@ -320,6 +321,7 @@ class Schema(AbstractSchema): schema=Table(dict( id=TextColumn(), git_commit_hash=TextColumn(), + git_commit_generation=IntegerColumn(), patchset_hash=TextColumn(), origin=TextColumn(), git_repository_url=TextColumn(), @@ -478,6 +480,36 @@ class Schema(AbstractSchema): misc=JSONColumn(), )), ), + transition=dict( + statement="SELECT\n" + " NULL AS id,\n" + " NULL AS version,\n" + " NULL AS origin,\n" + " NULL AS issue_id,\n" + " NULL AS issue_version,\n" + " NULL AS appearance,\n" + " NULL AS revision_before_git_commit_hash,\n" + " NULL AS revision_before_patchset_hash,\n" + " NULL AS revision_after_git_commit_hash,\n" + " NULL AS revision_after_patchset_hash,\n" + " NULL AS comment,\n" + " NULL AS misc\n" + "WHERE FALSE", + schema=Table(dict( + id=TextColumn(), + version=IntegerColumn(), + origin=TextColumn(), + issue_id=TextColumn(), + issue_version=IntegerColumn(), + appearance=BoolColumn(), + revision_before_git_commit_hash=TextColumn(), + revision_before_patchset_hash=TextColumn(), + revision_after_git_commit_hash=TextColumn(), + revision_after_patchset_hash=TextColumn(), + comment=TextColumn(), + misc=JSONColumn(), + )), + ), ) def init(self): diff --git a/kcidb/db/postgresql/v04_07.py b/kcidb/db/postgresql/v04_07.py new file mode 100644 index 00000000..6421e592 --- /dev/null +++ b/kcidb/db/postgresql/v04_07.py @@ -0,0 +1,175 @@ +"""Kernel CI report database - PostgreSQL schema v4.7""" + +import kcidb.io as io +from kcidb.misc import merge_dicts +from kcidb.db.postgresql.schema import Index +from .v04_02 import TIMESTAMP_COLUMN +from .v04_06 import Schema as PreviousSchema +from .schema import \ + Table, Constraint, BoolColumn, IntegerColumn, TextColumn, JSONColumn + + +# It's OK, pylint: disable=too-many-ancestors +class Schema(PreviousSchema): + """PostgreSQL database schema v4.7""" + + # The schema's version. + version = (4, 7) + # The I/O schema the database schema supports + io = io.schema.V4_4 + + # A map of table names and Table constructor arguments + # For use by descendants + TABLES_ARGS = merge_dicts( + PreviousSchema.TABLES_ARGS, + checkouts=dict( + columns=merge_dicts( + PreviousSchema.TABLES_ARGS["checkouts"]["columns"], + git_commit_generation=IntegerColumn(), + ), + ), + transitions=dict( + columns={ + "_timestamp": TIMESTAMP_COLUMN, + "id": TextColumn(constraint=Constraint.NOT_NULL), + "version": IntegerColumn(constraint=Constraint.NOT_NULL), + "origin": TextColumn(constraint=Constraint.NOT_NULL), + "issue_id": TextColumn(constraint=Constraint.NOT_NULL), + "issue_version": + IntegerColumn(constraint=Constraint.NOT_NULL), + "appearance": BoolColumn(), + "revision_before.git_commit_hash": TextColumn(), + "revision_before.patchset_hash": TextColumn(), + "revision_after.git_commit_hash": TextColumn(), + "revision_after.patchset_hash": TextColumn(), + "comment": TextColumn(), + "misc": JSONColumn(), + }, + primary_key=["id", "version"] + ), + ) + + # A map of table names and schemas + TABLES = { + name: Table(**args) for name, args in TABLES_ARGS.items() + } + + # A map of index names and schemas + INDEXES = merge_dicts(PreviousSchema.INDEXES, dict( + checkouts_git_commit_generation=Index( + "checkouts", ["git_commit_generation"] + ), + transitions__timestamp=Index("transitions", ["_timestamp"]), + transitions_origin=Index("transitions", ["origin"]), + transitions_issue_id=Index("transitions", ["issue_id"]), + transitions_revision_before_git_commit_hash_patchset_hash=Index( + "transitions", + ["revision_before_git_commit_hash", + "revision_before_patchset_hash"] + ), + transitions_revision_after_git_commit_hash_patchset_hash=Index( + "transitions", + ["revision_after_git_commit_hash", + "revision_after_patchset_hash"] + ), + )) + + # Queries and their columns for each type of raw object-oriented data. + # Both should have columns in the same order. + OO_QUERIES = merge_dicts( + PreviousSchema.OO_QUERIES, + checkout=merge_dicts( + PreviousSchema.OO_QUERIES["checkout"], + statement="SELECT\n" + " id,\n" + " git_commit_hash,\n" + # Implement this column + " git_commit_generation,\n" + " patchset_hash,\n" + " origin,\n" + " git_repository_url,\n" + " git_repository_branch,\n" + " tree_name,\n" + " message_id,\n" + " start_time,\n" + " log_url,\n" + " log_excerpt,\n" + " comment,\n" + " valid,\n" + " misc\n" + "FROM checkouts", + ), + # Implement transitions + transition=merge_dicts( + PreviousSchema.OO_QUERIES["transition"], + statement="SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " revision_before_git_commit_hash,\n" + " revision_before_patchset_hash,\n" + " revision_after_git_commit_hash,\n" + " revision_after_patchset_hash,\n" + " comment,\n" + " misc\n" + "FROM (\n" + " SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " revision_before_git_commit_hash,\n" + " revision_before_patchset_hash,\n" + " revision_after_git_commit_hash,\n" + " revision_after_patchset_hash,\n" + " comment,\n" + " misc,\n" + " ROW_NUMBER() OVER (\n" + " PARTITION BY id\n" + " ORDER BY version DESC\n" + " ) AS precedence\n" + " FROM transitions\n" + ") AS prioritized_transitions\n" + "WHERE precedence = 1", + ), + ) + + @classmethod + def _inherit(cls, conn): + """ + Inerit the database data from the previous schema version (if any). + + Args: + conn: Connection to the database to inherit. The database must + comply with the previous version of the schema. + """ + assert isinstance(conn, cls.Connection) + with conn, conn.cursor() as cursor: + # Add the git_commit_generation column to checkouts + column = cls.TABLES["checkouts"].columns["git_commit_generation"] + cursor.execute(f""" + ALTER TABLE checkouts ADD COLUMN {column.format_def()} + """) + # Create new tables + for table_name, table_schema in cls.TABLES.items(): + if table_name not in PreviousSchema.TABLES: + try: + cursor.execute(table_schema.format_create(table_name)) + except Exception as exc: + raise Exception( + f"Failed creating table {table_name!r}" + ) from exc + # Create new indexes + for index_name, index_schema in cls.INDEXES.items(): + if index_name not in PreviousSchema.INDEXES: + try: + cursor.execute(index_schema.format_create(index_name)) + except Exception as exc: + raise Exception( + f"Failed creating index {index_name!r}" + ) from exc diff --git a/kcidb/db/sqlite/__init__.py b/kcidb/db/sqlite/__init__.py index 4df1d331..7819b473 100644 --- a/kcidb/db/sqlite/__init__.py +++ b/kcidb/db/sqlite/__init__.py @@ -2,7 +2,7 @@ import textwrap from kcidb.db.schematic import Driver as SchematicDriver -from kcidb.db.sqlite.v04_02 import Schema as LatestSchema +from kcidb.db.sqlite.v04_03 import Schema as LatestSchema class Driver(SchematicDriver): diff --git a/kcidb/db/sqlite/v04_00.py b/kcidb/db/sqlite/v04_00.py index f7f55da8..6cbfe035 100644 --- a/kcidb/db/sqlite/v04_00.py +++ b/kcidb/db/sqlite/v04_00.py @@ -266,6 +266,7 @@ class Schema(AbstractSchema): statement="SELECT\n" " id,\n" " git_commit_hash,\n" + " NULL AS git_commit_generation,\n" " patchset_hash,\n" " origin,\n" " git_repository_url,\n" @@ -282,6 +283,7 @@ class Schema(AbstractSchema): schema=Table(dict( id=TextColumn(), git_commit_hash=TextColumn(), + git_commit_generation=IntegerColumn(), patchset_hash=TextColumn(), origin=TextColumn(), git_repository_url=TextColumn(), @@ -440,6 +442,36 @@ class Schema(AbstractSchema): misc=JSONColumn(), )), ), + transition=dict( + statement="SELECT\n" + " NULL AS id,\n" + " NULL AS version,\n" + " NULL AS origin,\n" + " NULL AS issue_id,\n" + " NULL AS issue_version,\n" + " NULL AS appearance,\n" + " NULL AS revision_before_git_commit_hash,\n" + " NULL AS revision_before_patchset_hash,\n" + " NULL AS revision_after_git_commit_hash,\n" + " NULL AS revision_after_patchset_hash,\n" + " NULL AS comment,\n" + " NULL AS misc\n" + "WHERE 0", + schema=Table(dict( + id=TextColumn(), + version=IntegerColumn(), + origin=TextColumn(), + issue_id=TextColumn(), + issue_version=IntegerColumn(), + appearance=BoolColumn(), + revision_before_git_commit_hash=TextColumn(), + revision_before_patchset_hash=TextColumn(), + revision_after_git_commit_hash=TextColumn(), + revision_after_patchset_hash=TextColumn(), + comment=TextColumn(), + misc=JSONColumn(), + )), + ), ) def init(self): diff --git a/kcidb/db/sqlite/v04_03.py b/kcidb/db/sqlite/v04_03.py new file mode 100644 index 00000000..690f779e --- /dev/null +++ b/kcidb/db/sqlite/v04_03.py @@ -0,0 +1,159 @@ +"""Kernel CI report database - SQLite schema v4.3""" + +import logging +import kcidb.io as io +from kcidb.misc import merge_dicts +from kcidb.db.sqlite.schema import \ + Constraint, BoolColumn, IntegerColumn, TextColumn, \ + JSONColumn, Table +from .v04_02 import Schema as PreviousSchema, TIMESTAMP_COLUMN + +# Module's logger +LOGGER = logging.getLogger(__name__) + + +class Schema(PreviousSchema): + """SQLite database schema v4.3""" + + # The schema's version. + version = (4, 3) + # The I/O schema the database schema supports + io = io.schema.V4_4 + + # A map of table names and Table constructor arguments + # For use by descendants + TABLES_ARGS = merge_dicts( + PreviousSchema.TABLES_ARGS, + checkouts=dict( + columns=merge_dicts( + PreviousSchema.TABLES_ARGS["checkouts"]["columns"], + git_commit_generation=IntegerColumn(), + ), + ), + transitions=dict( + columns={ + "_timestamp": TIMESTAMP_COLUMN, + "id": TextColumn(constraint=Constraint.NOT_NULL), + "version": IntegerColumn(constraint=Constraint.NOT_NULL), + "origin": TextColumn(constraint=Constraint.NOT_NULL), + "issue_id": TextColumn(constraint=Constraint.NOT_NULL), + "issue_version": + IntegerColumn(constraint=Constraint.NOT_NULL), + "appearance": BoolColumn(), + "revision_before.git_commit_hash": TextColumn(), + "revision_before.patchset_hash": TextColumn(), + "revision_after.git_commit_hash": TextColumn(), + "revision_after.patchset_hash": TextColumn(), + "comment": TextColumn(), + "misc": JSONColumn(), + }, + primary_key=["id", "version"] + ), + ) + + # A map of table names and schemas + TABLES = { + name: Table(**args) for name, args in TABLES_ARGS.items() + } + + # Queries and their columns for each type of raw object-oriented data. + # Both should have columns in the same order. + OO_QUERIES = merge_dicts( + PreviousSchema.OO_QUERIES, + checkout=merge_dicts( + PreviousSchema.OO_QUERIES["checkout"], + statement="SELECT\n" + " id,\n" + " git_commit_hash,\n" + # Add this column + " git_commit_generation,\n" + " patchset_hash,\n" + " origin,\n" + " git_repository_url,\n" + " git_repository_branch,\n" + " tree_name,\n" + " message_id,\n" + " start_time,\n" + " log_url,\n" + " log_excerpt,\n" + " comment,\n" + " valid,\n" + " misc\n" + "FROM checkouts", + ), + # Add transitions + transition=merge_dicts( + PreviousSchema.OO_QUERIES["transition"], + statement="SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " \"revision_before.git_commit_hash\" AS " + "revision_before_git_commit_hash,\n" + " \"revision_before.patchset_hash\" AS " + "revision_before_patchset_hash,\n" + " \"revision_after.git_commit_hash\" AS " + "revision_after_git_commit_hash,\n" + " \"revision_after.patchset_hash\" AS " + "revision_after_patchset_hash,\n" + " comment,\n" + " misc\n" + "FROM (\n" + " SELECT\n" + " id,\n" + " version,\n" + " origin,\n" + " issue_id,\n" + " issue_version,\n" + " appearance,\n" + " \"revision_before.git_commit_hash\",\n" + " \"revision_before.patchset_hash\",\n" + " \"revision_after.git_commit_hash\",\n" + " \"revision_after.patchset_hash\",\n" + " comment,\n" + " misc,\n" + " ROW_NUMBER() OVER (\n" + " PARTITION BY id\n" + " ORDER BY version DESC\n" + " ) AS precedence\n" + " FROM transitions\n" + ") AS prioritized_transitions\n" + "WHERE precedence = 1", + ), + ) + + @classmethod + def _inherit(cls, conn): + """ + Inerit the database data from the previous schema version (if any). + + Args: + conn: Connection to the database to inherit. The database must + comply with the previous version of the schema. + """ + assert isinstance(conn, cls.Connection) + with conn: + cursor = conn.cursor() + try: + # Add the git_commit_generation column to checkouts + column = cls.TABLES["checkouts"]. \ + columns["git_commit_generation"] + cursor.execute(f""" + ALTER TABLE checkouts ADD COLUMN {column.format_def()} + """) + # Create new tables + for table_name, table_schema in cls.TABLES.items(): + if table_name not in PreviousSchema.TABLES: + try: + cursor.execute( + table_schema.format_create(table_name) + ) + except Exception as exc: + raise Exception( + f"Failed creating table {table_name!r}" + ) from exc + finally: + cursor.close() diff --git a/kcidb/io.py b/kcidb/io.py index ed0dcca0..cb776b18 100644 --- a/kcidb/io.py +++ b/kcidb/io.py @@ -6,4 +6,4 @@ from kcidb_io import * # noqa: F403 # The I/O schema version used by KCIDB -SCHEMA = schema.V4_3 # noqa: F405 +SCHEMA = schema.V4_4 # noqa: F405 diff --git a/kcidb/oo/__init__.py b/kcidb/oo/__init__.py index 2d828dfa..18781109 100644 --- a/kcidb/oo/__init__.py +++ b/kcidb/oo/__init__.py @@ -698,6 +698,10 @@ class Incident(Object): """An OO-representation of an incident""" +class Transition(Object): + """An OO-representation of a transition""" + + # A map of object type names and Object-derived classes handling their data CLASSES = dict( revision=Revision, @@ -707,6 +711,7 @@ class Incident(Object): bug=Bug, issue=Issue, incident=Incident, + transition=Transition, ) assert set(CLASSES) == set(SCHEMA.types) diff --git a/kcidb/orm/data.py b/kcidb/orm/data.py index bf88fc9b..662f6421 100644 --- a/kcidb/orm/data.py +++ b/kcidb/orm/data.py @@ -305,6 +305,8 @@ def format_dot(self): # Latest I/O schema shared definitions _DEFS = io.SCHEMA.json['$defs'] +# Revision ID properties from the current I/O schema +_REVISION_ID = _DEFS['revision_id']['properties'] # Checkout properties from the current I/O schema _CHECKOUT = _DEFS['checkout']['properties'] # Build properties from the current I/O schema @@ -317,6 +319,8 @@ def format_dot(self): _ISSUE_CULPRIT = _ISSUE['culprit']['properties'] # Incident properties from the current I/O schema _INCIDENT = _DEFS['incident']['properties'] +# Transition properties from the current I/O schema +_TRANSITION = _DEFS['transition']['properties'] # Test environment properties from the current I/O schema _TEST_ENVIRONMENT = _TEST['environment']['properties'] @@ -342,6 +346,7 @@ def format_dot(self): field_json_schemas=dict( id=_CHECKOUT['id'], git_commit_hash=_CHECKOUT['git_commit_hash'], + git_commit_generation=_CHECKOUT['git_commit_generation'], patchset_hash=_CHECKOUT['patchset_hash'], origin=_CHECKOUT['origin'], git_repository_url=_CHECKOUT['git_repository_url'], @@ -445,6 +450,7 @@ def format_dot(self): id_fields=("id",), children=dict( incident=("issue_id",), + transition=("issue_id",), ), ), incident=dict( @@ -461,6 +467,28 @@ def format_dot(self): required_fields={'id', 'origin', 'issue_id', 'issue_version'}, id_fields=("id",), ), + transition=dict( + field_json_schemas=dict( + id=_TRANSITION['id'], + version=_TRANSITION['version'], + origin=_TRANSITION['origin'], + issue_id=_TRANSITION['issue_id'], + issue_version=_TRANSITION['issue_version'], + appearance=_TRANSITION['appearance'], + revision_before_git_commit_hash=_REVISION_ID[ + 'git_commit_hash' + ], + revision_before_patchset_hash=_REVISION_ID['patchset_hash'], + revision_after_git_commit_hash=_REVISION_ID[ + 'git_commit_hash' + ], + revision_after_patchset_hash=_REVISION_ID['patchset_hash'], + comment=_TRANSITION['comment'], + misc=_TRANSITION['misc'], + ), + required_fields={'id', 'origin', 'issue_id', 'issue_version'}, + id_fields=("id",), + ), ) ) diff --git a/kcidb/test_db.py b/kcidb/test_db.py index c4e8b527..08c0f9a8 100644 --- a/kcidb/test_db.py +++ b/kcidb/test_db.py @@ -19,7 +19,10 @@ def test_schemas_main(): """Check kcidb-db-schemas works""" argv = ["kcidb.db.schemas_main", "-d", "sqlite::memory:"] assert_executes("", *argv, - stdout_re=r"4\.0: 4\.0\n4\.1: 4\.2\n4\.2: 4\.3\n") + stdout_re=r"4\.0: 4\.0\n" + r"4\.1: 4\.2\n" + r"4\.2: 4\.3\n" + r"4\.3: 4\.4\n") def test_reset(clean_database): @@ -234,7 +237,7 @@ def test_load_main(): # I/O data containing all possible fields COMPREHENSIVE_IO_DATA = { - **kcidb.io.SCHEMA.new(), + **kcidb.io.schema.V4_4.new(), "checkouts": [ dict( id="origin:1", @@ -243,6 +246,7 @@ def test_load_main(): git_repository_url="https://git.kernel.org/pub/scm/" "linux/kernel/git/torvalds/linux.git", git_commit_hash="ef2b753c38bc9c0d1eea84e29a6bb6f9e776d0e3", + git_commit_generation=1917, git_commit_name="v5.8-rc7-98-g7dc6fd0f3b84", git_repository_branch="master", patchset_files=[ @@ -374,7 +378,31 @@ def test_load_main(): baz=42 ), ) - ] + ], + "transitions": [ + dict( + id="maestro:898a90980d09897cb098e", + version=2, + origin="maestro", + issue_id="redhat:878234322", + issue_version=3, + appearance=True, + revision_before=dict( + git_commit_hash="00c288790dffb779e89bb42" + "e36773a00a740e0cc", + patchset_hash="" + ), + revision_after=dict( + git_commit_hash="5acb9c2a7bc836e9e5172bb" + "cd2311499c5b4e5f1", + patchset_hash="" + ), + comment="It's breaking!", + misc=dict( + too="bad", + ), + ) + ], } @@ -742,6 +770,7 @@ def test_upgrade(clean_database): 'comment': None, 'git_commit_hash': 'f00af9d68ed146b47fdbfe91134fcf04c36e6d78', + 'git_commit_generation': None, 'git_repository_branch': 'android-mainline', 'git_repository_url': 'https://android.googlesource.com/kernel/common.git', @@ -828,6 +857,7 @@ def test_upgrade(clean_database): 'bug': [], 'issue': [], 'incident': [], + 'transition': [], } ), kcidb.io.schema.V4_0: dict( @@ -869,6 +899,7 @@ def test_upgrade(clean_database): "comment": None, "git_commit_hash": "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + 'git_commit_generation': None, "git_repository_branch": None, "git_repository_url": None, "id": @@ -927,6 +958,7 @@ def test_upgrade(clean_database): "bug": [], "issue": [], "incident": [], + 'transition': [], } ), kcidb.io.schema.V4_2: dict( @@ -995,6 +1027,7 @@ def test_upgrade(clean_database): "comment": None, "git_commit_hash": "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + 'git_commit_generation': None, "git_repository_branch": None, "git_repository_url": None, "id": @@ -1090,6 +1123,7 @@ def test_upgrade(clean_database): "test_id": "google:google.org:a19di3j5h67f8d9475f26v11", }], + 'transition': [], } ), kcidb.io.schema.V4_3: dict( @@ -1158,6 +1192,7 @@ def test_upgrade(clean_database): "comment": None, "git_commit_hash": "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + 'git_commit_generation': None, "git_repository_branch": None, "git_repository_url": None, "id": @@ -1253,6 +1288,206 @@ def test_upgrade(clean_database): "test_id": "google:google.org:a19di3j5h67f8d9475f26v11", }], + 'transition': [], + } + ), + kcidb.io.schema.V4_4: dict( + io={ + "version": {"major": 4, "minor": 4}, + "checkouts": [{ + "id": "_:kernelci:5acb9c2a7bc836e" + "9e5172bbcd2311499c5b4e5f1", + "origin": "kernelci", + "git_commit_hash": "5acb9c2a7bc836e9e5172bb" + "cd2311499c5b4e5f1", + "git_commit_generation": 1984, + "git_commit_name": "v5.15-4077-g5acb9c2a7bc8", + "patchset_hash": "" + }], + "builds": [{ + "id": "google:google.org:a1d993c3n4c448b2j0l1hbf1", + "origin": "google", + "checkout_id": "_:google:bd355732283c23a365f7c" + "55206c0385100d1c389" + }], + "tests": [{ + "id": "google:google.org:a19di3j5h67f8d9475f26v11", + "build_id": "google:google.org:a1d993c3n4c448b2" + "j0l1hbf1", + "origin": "google", + "status": "MISS", + }], + "issues": [{ + "id": "redhat:878234322", + "version": 3, + "origin": "redhat", + "report_url": + "https://bugzilla.redhat.com/show_bug.cgi" + "?id=873123", + "report_subject": + "(cups-usb-quirks) - usb printer doesn't print " + "(usblp0: USB Bidirectional printer dev)", + "culprit": { + "code": True, + "tool": False, + "harness": False, + }, + "comment": "Match USB Bidirectional printer dev", + }], + "incidents": [{ + "id": "redhat:2340981234098123409382", + "issue_id": "redhat:878234322", + "issue_version": 3, + "origin": "redhat", + "test_id": + "google:google.org:a19di3j5h67f8d9475f26v11", + "present": True, + }], + "transitions": [{ + "id": "maestro:898a90980d09897cb098e", + "version": 2, + "origin": "maestro", + "issue_id": "redhat:878234322", + "issue_version": 3, + "appearance": True, + "revision_before": { + "git_commit_hash": "00c288790dffb779e89bb42" + "e36773a00a740e0cc", + "patchset_hash": "" + }, + "revision_after": { + "git_commit_hash": "5acb9c2a7bc836e9e5172bb" + "cd2311499c5b4e5f1", + "patchset_hash": "" + }, + }] + }, + oo={ + "revision": [{ + "contacts": None, + "git_commit_hash": + "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + "git_commit_name": + "v5.15-4077-g5acb9c2a7bc8", + "patchset_files": None, + "patchset_hash": "", + }], + "checkout": [{ + "comment": None, + "git_commit_hash": + "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + 'git_commit_generation': 1984, + "git_repository_branch": None, + "git_repository_url": None, + "id": + "_:kernelci:" + "5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1", + "log_excerpt": None, + "log_url": None, + "message_id": None, + "misc": None, + "origin": "kernelci", + "patchset_hash": "", + "start_time": None, + "tree_name": None, + "valid": None, + }], + "build": [{ + "architecture": None, + "checkout_id": + "_:google:" + "bd355732283c23a365f7c55206c0385100d1c389", + "command": None, + "comment": None, + "compiler": None, + "config_name": None, + "config_url": None, + "duration": None, + "id": "google:google.org:a1d993c3n4c448b2j0l1hbf1", + "input_files": None, + "log_excerpt": None, + "log_url": None, + "misc": None, + "origin": "google", + "output_files": None, + "start_time": None, + "valid": None, + }], + "test": [{ + "build_id": + "google:google.org:a1d993c3n4c448b2j0l1hbf1", + "comment": None, + "duration": None, + "environment_comment": None, + "environment_misc": None, + "id": + "google:google.org:a19di3j5h67f8d9475f26v11", + "log_excerpt": None, + "log_url": None, + "misc": None, + "origin": "google", + "output_files": None, + "path": None, + "start_time": None, + "status": "MISS", + "waived": None, + }], + "bug": [{ + "culprit_code": True, + "culprit_tool": False, + "culprit_harness": False, + "url": + "https://bugzilla.redhat.com/show_bug.cgi" + "?id=873123", + "subject": + "(cups-usb-quirks) - usb printer doesn't print " + "(usblp0: USB Bidirectional printer dev)", + }], + "issue": [{ + "comment": "Match USB Bidirectional printer dev", + "id": "redhat:878234322", + "misc": None, + "origin": "redhat", + "report_url": + "https://bugzilla.redhat.com/show_bug.cgi?" + "id=873123", + "report_subject": + "(cups-usb-quirks) - usb printer doesn't print " + "(usblp0: USB Bidirectional printer dev)", + "culprit_code": True, + "culprit_tool": False, + "culprit_harness": False, + "build_valid": None, + "test_status": None, + "version": 3, + }], + "incident": [{ + "build_id": None, + "comment": None, + "id": "redhat:2340981234098123409382", + "issue_id": "redhat:878234322", + "issue_version": 3, + "misc": None, + "origin": "redhat", + "test_id": + "google:google.org:a19di3j5h67f8d9475f26v11", + }], + "transition": [{ + 'appearance': True, + 'comment': None, + 'id': 'maestro:898a90980d09897cb098e', + 'issue_id': 'redhat:878234322', + 'issue_version': 3, + 'misc': None, + 'origin': 'maestro', + 'revision_after_git_commit_hash': + '5acb9c2a7bc836e9e5172bbcd2311499c5b4e5f1', + 'revision_after_patchset_hash': '', + 'revision_before_git_commit_hash': + '00c288790dffb779e89bb42e36773a00a740e0cc', + 'revision_before_patchset_hash': '', + 'version': 2 + }], } ), } @@ -1274,11 +1509,17 @@ def test_upgrade(clean_database): # Check upgrade went well # You're wrong, pylint: disable=unsubscriptable-object assert database.oo_query(kcidb.orm.query.Pattern.parse(">*#")) == \ - last_params["oo"] + last_params["oo"], \ + f"Unexpected OO query result after " \ + f"{last_io_version} -> {io_version} DB upgrade" upgraded_io = io_version.upgrade(last_params["io"]) - assert database.dump(with_metadata=False) == upgraded_io + assert database.dump(with_metadata=False) == upgraded_io, \ + f"Unexpected IO dump after " \ + f"{last_io_version} -> {io_version} DB upgrade" assert database.query(io_version.get_ids(upgraded_io)) == \ - upgraded_io + upgraded_io, \ + f"Unexpected IO query result after " \ + f"{last_io_version} -> {io_version} DB upgrade" # For each data's I/O version and parameters for load_io_version, load_params in io_version_params.items(): @@ -1307,12 +1548,18 @@ def test_upgrade(clean_database): # Check we can query it in various ways upgraded_io = io_version.upgrade(load_params["io"]) - assert database.dump(with_metadata=False) == upgraded_io + assert database.dump(with_metadata=False) == upgraded_io, \ + f"Unexpected dump result after loading {load_io_version} " \ + f"data into {io_version} database" assert database.query(io_version.get_ids(upgraded_io)) == \ - upgraded_io + upgraded_io, \ + f"Unexpected I/O query result after loading " \ + f"{load_io_version} data into {io_version} database" assert \ database.oo_query(kcidb.orm.query.Pattern.parse(">*#")) == \ - load_params["oo"] + load_params["oo"], \ + f"Unexpected OO query result after loading " \ + f"{load_io_version} data into {io_version} database" # Remeber this I/O version and its parameters for the next round last_io_version = io_version diff --git a/kcidb/test_orm.py b/kcidb/test_orm.py index 9815bca6..9bb8a7c8 100644 --- a/kcidb/test_orm.py +++ b/kcidb/test_orm.py @@ -114,6 +114,18 @@ required_fields={'id', 'origin', 'issue_id', 'issue_version'}, id_fields=("id",), ), + transition=dict( + field_json_schemas=dict( + id=dict(type="string"), + version=dict(type="integer"), + origin=dict(type="string"), + issue_id=dict(type="string"), + issue_version=dict(type="integer"), + appearance=dict(type="boolean"), + ), + required_fields={'id', 'origin', 'issue_id', 'issue_version'}, + id_fields=("id",), + ), ) ) diff --git a/requirements.txt b/requirements.txt index 9d13b9b9..3303b028 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ jinja2 python-dateutil cached-property jq@git+https://github.com/kernelci/jq.py.git@1.7.0.post1 -kcidb-io@git+https://github.com/kernelci/kcidb-io.git +kcidb-io@git+https://github.com/kernelci/kcidb-io.git@add_schema_v4_4 diff --git a/setup.py b/setup.py index c9254887..0fc93d3f 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ "python-dateutil", "cached-property", "jq@git+https://github.com/kernelci/jq.py.git@1.7.0.post1", - "kcidb-io@git+https://github.com/kernelci/kcidb-io.git", + "kcidb-io@git+https://github.com/kernelci/kcidb-io.git@add_schema_v4_4", ], extras_require=dict( dev=[ diff --git a/test_main.py b/test_main.py index 5e8f67ee..e4e5f084 100644 --- a/test_main.py +++ b/test_main.py @@ -263,6 +263,11 @@ def test_purge_db(empty_deployment): issue_id="origin:1", issue_version=1, _timestamp=str_before )], + transitions=[dict( + id="origin:1", version=1, origin="origin", + issue_id="origin:1", issue_version=1, + _timestamp=str_before + )], ) timestamp_after = timestamp_before + timedelta(microseconds=1) @@ -293,6 +298,11 @@ def test_purge_db(empty_deployment): issue_id="origin:2", issue_version=1, _timestamp=str_after )], + transitions=[dict( + id="origin:2", version=1, origin="origin", + issue_id="origin:2", issue_version=1, + _timestamp=str_after + )], ) def filter_test_data(data):