Edit on GitHub

sqlmesh.core.schema_diff

 1from __future__ import annotations
 2
 3import typing as t
 4from enum import Enum, auto
 5
 6from sqlmesh.utils.pydantic import PydanticModel
 7
 8if t.TYPE_CHECKING:
 9    from sqlmesh.core.engine_adapter import EngineAdapter
10
11
12class SchemaDeltaOp(Enum):
13    ADD = auto()
14    DROP = auto()
15    ALTER_TYPE = auto()
16
17
18class SchemaDelta(PydanticModel):
19    column_name: str
20    column_type: str
21    op: SchemaDeltaOp
22
23    @classmethod
24    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
25        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)
26
27    @classmethod
28    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
29        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)
30
31    @classmethod
32    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
33        return cls(
34            column_name=column_name,
35            column_type=column_type,
36            op=SchemaDeltaOp.ALTER_TYPE,
37        )
38
39
40class SchemaDiffCalculator:
41    """Calculates the difference between table schemas.
42
43    Args:
44        engine_adapter: The engine adapter.
45        is_type_transition_allowed: The predicate which accepts the source type and the target type
46            and returns True if the transition from source to target is allowed and False otherwise.
47            Default: no type transitions are allowed.
48    """
49
50    def __init__(
51        self,
52        engine_adapter: EngineAdapter,
53        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
54    ):
55        self.engine_adapter = engine_adapter
56        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)
57
58    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
59        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
60        brings its schema in correspondence with the schema of the second table.
61
62        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
63
64        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
65        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
66
67        Args:
68            apply_to_table: The name of the table to which deltas will be applied.
69            schema_from_table: The schema of this table will be used for comparison.
70
71        Returns:
72            The list of deltas.
73        """
74        to_schema = self.engine_adapter.columns(apply_to_table)
75        from_schema = self.engine_adapter.columns(schema_from_table)
76
77        result = []
78
79        for to_column_name, to_column_type in to_schema.items():
80            from_column_type = from_schema.get(to_column_name)
81            from_column_type = from_column_type.upper() if from_column_type else None
82            if from_column_type != to_column_type.upper():
83                if from_column_type and self.is_type_transition_allowed(
84                    to_column_type, from_column_type
85                ):
86                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
87                else:
88                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
89                    if from_column_type:
90                        result.append(SchemaDelta.add(to_column_name, from_column_type))
91
92        for from_column_name, from_column_type in from_schema.items():
93            if from_column_name not in to_schema:
94                result.append(SchemaDelta.add(from_column_name, from_column_type))
95
96        return result
class SchemaDeltaOp(enum.Enum):
13class SchemaDeltaOp(Enum):
14    ADD = auto()
15    DROP = auto()
16    ALTER_TYPE = auto()

An enumeration.

ADD = <SchemaDeltaOp.ADD: 1>
DROP = <SchemaDeltaOp.DROP: 2>
ALTER_TYPE = <SchemaDeltaOp.ALTER_TYPE: 3>
Inherited Members
enum.Enum
name
value
class SchemaDelta(sqlmesh.utils.pydantic.PydanticModel):
19class SchemaDelta(PydanticModel):
20    column_name: str
21    column_type: str
22    op: SchemaDeltaOp
23
24    @classmethod
25    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
26        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)
27
28    @classmethod
29    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
30        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)
31
32    @classmethod
33    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
34        return cls(
35            column_name=column_name,
36            column_type=column_type,
37            op=SchemaDeltaOp.ALTER_TYPE,
38        )
@classmethod
def add( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta:
24    @classmethod
25    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
26        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)
@classmethod
def drop( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta:
28    @classmethod
29    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
30        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)
@classmethod
def alter_type( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta:
32    @classmethod
33    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
34        return cls(
35            column_name=column_name,
36            column_type=column_type,
37            op=SchemaDeltaOp.ALTER_TYPE,
38        )
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class SchemaDiffCalculator:
41class SchemaDiffCalculator:
42    """Calculates the difference between table schemas.
43
44    Args:
45        engine_adapter: The engine adapter.
46        is_type_transition_allowed: The predicate which accepts the source type and the target type
47            and returns True if the transition from source to target is allowed and False otherwise.
48            Default: no type transitions are allowed.
49    """
50
51    def __init__(
52        self,
53        engine_adapter: EngineAdapter,
54        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
55    ):
56        self.engine_adapter = engine_adapter
57        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)
58
59    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
60        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
61        brings its schema in correspondence with the schema of the second table.
62
63        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
64
65        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
66        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
67
68        Args:
69            apply_to_table: The name of the table to which deltas will be applied.
70            schema_from_table: The schema of this table will be used for comparison.
71
72        Returns:
73            The list of deltas.
74        """
75        to_schema = self.engine_adapter.columns(apply_to_table)
76        from_schema = self.engine_adapter.columns(schema_from_table)
77
78        result = []
79
80        for to_column_name, to_column_type in to_schema.items():
81            from_column_type = from_schema.get(to_column_name)
82            from_column_type = from_column_type.upper() if from_column_type else None
83            if from_column_type != to_column_type.upper():
84                if from_column_type and self.is_type_transition_allowed(
85                    to_column_type, from_column_type
86                ):
87                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
88                else:
89                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
90                    if from_column_type:
91                        result.append(SchemaDelta.add(to_column_name, from_column_type))
92
93        for from_column_name, from_column_type in from_schema.items():
94            if from_column_name not in to_schema:
95                result.append(SchemaDelta.add(from_column_name, from_column_type))
96
97        return result

Calculates the difference between table schemas.

Arguments:
  • engine_adapter: The engine adapter.
  • is_type_transition_allowed: The predicate which accepts the source type and the target type and returns True if the transition from source to target is allowed and False otherwise. Default: no type transitions are allowed.
SchemaDiffCalculator( engine_adapter: sqlmesh.core.engine_adapter.base.EngineAdapter, is_type_transition_allowed: Optional[Callable[[str, str], bool]] = None)
51    def __init__(
52        self,
53        engine_adapter: EngineAdapter,
54        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
55    ):
56        self.engine_adapter = engine_adapter
57        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)
def calculate( self, apply_to_table: str, schema_from_table: str) -> List[sqlmesh.core.schema_diff.SchemaDelta]:
59    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
60        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
61        brings its schema in correspondence with the schema of the second table.
62
63        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
64
65        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
66        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
67
68        Args:
69            apply_to_table: The name of the table to which deltas will be applied.
70            schema_from_table: The schema of this table will be used for comparison.
71
72        Returns:
73            The list of deltas.
74        """
75        to_schema = self.engine_adapter.columns(apply_to_table)
76        from_schema = self.engine_adapter.columns(schema_from_table)
77
78        result = []
79
80        for to_column_name, to_column_type in to_schema.items():
81            from_column_type = from_schema.get(to_column_name)
82            from_column_type = from_column_type.upper() if from_column_type else None
83            if from_column_type != to_column_type.upper():
84                if from_column_type and self.is_type_transition_allowed(
85                    to_column_type, from_column_type
86                ):
87                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
88                else:
89                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
90                    if from_column_type:
91                        result.append(SchemaDelta.add(to_column_name, from_column_type))
92
93        for from_column_name, from_column_type in from_schema.items():
94            if from_column_name not in to_schema:
95                result.append(SchemaDelta.add(from_column_name, from_column_type))
96
97        return result

Calculates a list of schema deltas between the two tables, applying which in order to the first table brings its schema in correspondence with the schema of the second table.

Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.

Additionally the implementation currently doesn't differentiate between regular columns and partition ones. It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.

Arguments:
  • apply_to_table: The name of the table to which deltas will be applied.
  • schema_from_table: The schema of this table will be used for comparison.
Returns:

The list of deltas.