Skip to content

Commit 4ce818b

Browse files
committed
Refactor: consolidate _validate_concurrency into base class
Move the duplicated _validate_concurrency logic from _DeleteFiles and _OverwriteFiles into _SnapshotProducer. Extract _resolve_parent_snapshot and _resolve_starting_snapshot helpers for readability. _FastAppendFiles overrides with a no-op since appends do not require conflict validation. Also move property_as_int to a top-level import in table/__init__.py. Signed-off-by: Sotaro Hikita <bering1814@gmail.com>
1 parent 570ecbe commit 4ce818b

2 files changed

Lines changed: 61 additions & 98 deletions

File tree

pyiceberg/table/__init__.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@
9696
from pyiceberg.types import strtobool
9797
from pyiceberg.utils.concurrent import ExecutorFactory
9898
from pyiceberg.utils.config import Config
99-
from pyiceberg.utils.properties import property_as_bool
99+
from pyiceberg.utils.properties import property_as_bool, property_as_int
100100

101101
if TYPE_CHECKING:
102102
import bodo.pandas as bd
@@ -1073,8 +1073,6 @@ def commit_transaction(self) -> Table:
10731073
The table with the updates applied.
10741074
"""
10751075
if len(self._updates) > 0:
1076-
from pyiceberg.utils.properties import property_as_int
1077-
10781076
properties = self._table.metadata.properties
10791077
num_retries_val = property_as_int(
10801078
properties, TableProperties.COMMIT_NUM_RETRIES, TableProperties.COMMIT_NUM_RETRIES_DEFAULT

pyiceberg/table/update/snapshot.py

Lines changed: 60 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,63 @@ def _refresh_for_retry(self) -> None:
405405
self.commit_uuid = uuid.uuid4()
406406

407407
def _validate_concurrency(self) -> None:
408-
"""Validate that concurrent changes do not conflict with this operation. No-op by default."""
408+
"""Validate that concurrent changes do not conflict with this operation.
409+
410+
Checks isolation level and uses the conflict detection filter to determine
411+
whether concurrent commits introduced conflicting data or delete files.
412+
Subclasses that do not require validation (e.g. fast append) should override
413+
with a no-op.
414+
"""
415+
from pyiceberg.table import TableProperties
416+
from pyiceberg.table.snapshots import IsolationLevel
417+
from pyiceberg.table.update.validate import (
418+
_validate_added_data_files,
419+
_validate_deleted_data_files,
420+
_validate_no_new_delete_files,
421+
_validate_no_new_deletes_for_data_files,
422+
)
423+
424+
parent_snapshot = self._resolve_parent_snapshot()
425+
if parent_snapshot is None:
426+
return
427+
428+
starting_snapshot = self._resolve_starting_snapshot()
429+
430+
table = self._transaction._table
431+
isolation_level_str = table.metadata.properties.get(
432+
self._isolation_level_property, TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT
433+
)
434+
isolation_level = IsolationLevel(isolation_level_str)
435+
conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None
436+
437+
if isolation_level == IsolationLevel.SERIALIZABLE:
438+
_validate_added_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
439+
440+
if conflict_detection_filter is not None:
441+
_validate_no_new_delete_files(table, parent_snapshot, conflict_detection_filter, None, starting_snapshot)
442+
_validate_deleted_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
443+
444+
if self._deleted_data_files:
445+
_validate_no_new_deletes_for_data_files(
446+
table, parent_snapshot, conflict_detection_filter, self._deleted_data_files, starting_snapshot
447+
)
448+
449+
def _resolve_parent_snapshot(self) -> Snapshot | None:
450+
"""Resolve parent snapshot, raising ValidationException if ID is set but snapshot is missing."""
451+
if self._parent_snapshot_id is None:
452+
return None
453+
snapshot = self._transaction._table.metadata.snapshot_by_id(self._parent_snapshot_id)
454+
if snapshot is None:
455+
raise ValidationException(f"Cannot find parent snapshot {self._parent_snapshot_id} in table metadata")
456+
return snapshot
457+
458+
def _resolve_starting_snapshot(self) -> Snapshot:
459+
"""Resolve starting snapshot for the conflict detection window."""
460+
starting_id = self._starting_snapshot_id if self._starting_snapshot_id is not None else self._parent_snapshot_id
461+
snapshot = self._transaction._table.metadata.snapshot_by_id(starting_id)
462+
if snapshot is None:
463+
raise ValidationException(f"Cannot find starting snapshot {starting_id} in table metadata")
464+
return snapshot
409465

410466
def _build_partition_projection(self, spec_id: int) -> BooleanExpression:
411467
project = inclusive_projection(self.schema(), self.spec(spec_id), self._case_sensitive)
@@ -557,56 +613,11 @@ def _refresh_for_retry(self) -> None:
557613
if "_compute_deletes" in self.__dict__:
558614
del self.__dict__["_compute_deletes"]
559615

560-
def _validate_concurrency(self) -> None:
561-
"""Validate that concurrent changes do not conflict with this delete.
562-
563-
Note: This method is intentionally duplicated in _OverwriteFiles rather than
564-
extracted to the base class. While the logic is currently identical, Java Iceberg's
565-
BaseOverwriteFiles and BaseRowDelta have divergent validation. Keeping them separate
566-
makes it easier to add RowDelta-specific validation in the future.
567-
"""
568-
from pyiceberg.table import TableProperties
569-
from pyiceberg.table.snapshots import IsolationLevel
570-
from pyiceberg.table.update.validate import (
571-
_validate_added_data_files,
572-
_validate_deleted_data_files,
573-
_validate_no_new_delete_files,
574-
_validate_no_new_deletes_for_data_files,
575-
)
576-
577-
if self._parent_snapshot_id is None:
578-
return
579-
580-
table = self._transaction._table
581-
parent_snapshot = table.metadata.snapshot_by_id(self._parent_snapshot_id)
582-
if parent_snapshot is None:
583-
raise ValidationException(f"Cannot find parent snapshot {self._parent_snapshot_id} in table metadata")
584-
585-
starting_snapshot_id = self._starting_snapshot_id if self._starting_snapshot_id is not None else self._parent_snapshot_id
586-
starting_snapshot = table.metadata.snapshot_by_id(starting_snapshot_id)
587-
if starting_snapshot is None:
588-
raise ValidationException(f"Cannot find starting snapshot {starting_snapshot_id} in table metadata")
589-
590-
isolation_level_str = table.metadata.properties.get(
591-
self._isolation_level_property, TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT
592-
)
593-
isolation_level = IsolationLevel(isolation_level_str)
594-
conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None
595-
596-
if isolation_level == IsolationLevel.SERIALIZABLE:
597-
_validate_added_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
598-
599-
if conflict_detection_filter is not None:
600-
_validate_no_new_delete_files(table, parent_snapshot, conflict_detection_filter, None, starting_snapshot)
601-
_validate_deleted_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
602-
603-
if self._deleted_data_files:
604-
_validate_no_new_deletes_for_data_files(
605-
table, parent_snapshot, conflict_detection_filter, self._deleted_data_files, starting_snapshot
606-
)
607-
608616

609617
class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]):
618+
def _validate_concurrency(self) -> None:
619+
"""Appends do not conflict with other operations; skip validation."""
620+
610621
def _existing_manifests(self) -> list[ManifestFile]:
611622
"""To determine if there are any existing manifest files.
612623
@@ -777,52 +788,6 @@ def _get_entries(manifest: ManifestFile) -> list[ManifestEntry]:
777788
else:
778789
return []
779790

780-
def _validate_concurrency(self) -> None:
781-
"""Validate that concurrent changes do not conflict with this overwrite.
782-
783-
Note: See _DeleteFiles._validate_concurrency() for why this is intentionally
784-
duplicated rather than extracted to the base class.
785-
"""
786-
from pyiceberg.table import TableProperties
787-
from pyiceberg.table.snapshots import IsolationLevel
788-
from pyiceberg.table.update.validate import (
789-
_validate_added_data_files,
790-
_validate_deleted_data_files,
791-
_validate_no_new_delete_files,
792-
_validate_no_new_deletes_for_data_files,
793-
)
794-
795-
if self._parent_snapshot_id is None:
796-
return
797-
798-
table = self._transaction._table
799-
parent_snapshot = table.metadata.snapshot_by_id(self._parent_snapshot_id)
800-
if parent_snapshot is None:
801-
raise ValidationException(f"Cannot find parent snapshot {self._parent_snapshot_id} in table metadata")
802-
803-
starting_snapshot_id = self._starting_snapshot_id if self._starting_snapshot_id is not None else self._parent_snapshot_id
804-
starting_snapshot = table.metadata.snapshot_by_id(starting_snapshot_id)
805-
if starting_snapshot is None:
806-
raise ValidationException(f"Cannot find starting snapshot {starting_snapshot_id} in table metadata")
807-
808-
isolation_level_str = table.metadata.properties.get(
809-
self._isolation_level_property, TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT
810-
)
811-
isolation_level = IsolationLevel(isolation_level_str)
812-
conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None
813-
814-
if isolation_level == IsolationLevel.SERIALIZABLE:
815-
_validate_added_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
816-
817-
if conflict_detection_filter is not None:
818-
_validate_no_new_delete_files(table, parent_snapshot, conflict_detection_filter, None, starting_snapshot)
819-
_validate_deleted_data_files(table, parent_snapshot, conflict_detection_filter, starting_snapshot)
820-
821-
if self._deleted_data_files:
822-
_validate_no_new_deletes_for_data_files(
823-
table, parent_snapshot, conflict_detection_filter, self._deleted_data_files, starting_snapshot
824-
)
825-
826791

827792
class UpdateSnapshot:
828793
_transaction: Transaction

0 commit comments

Comments
 (0)