Sitelet https://github.com/Lanpart1/python-spanner/commit/3ca2689324406e0bd9a6b872eda4a23999115f0f
Skip to content

Commit 3ca2689

Browse files
harshachintagcf-owl-bot[bot]
harshachinta
and
gcf-owl-bot[bot]
authored
feat(spanner): add support for Proto Columns (googleapis#1084)
* feat: Proto Columns Feature (googleapis#909) * feat: adding proto autogenerated code changes for proto column feature * feat: add implementation for Proto columns DDL * feat: add implementation for Proto columns DML * feat: add implementation for Proto columns DQL * feat: add NoneType check during Proto deserialization * feat: add code changes for Proto DDL support * feat: add required proto files to execute samples and tests * feat: add sample snippets for Proto columns DDL * feat: add tests for proto columns ddl, dml, dql snippets * feat: code refactoring * feat: remove staging endpoint from snippets.py * feat: comment refactor * feat: add license file * feat: update proto column data in insertion sample * feat: move column_info argument to the end to avoid breaking code * feat: Proto column feature tests and samples (googleapis#921) * feat: add integration tests for Proto Columns * feat: add unit tests for Proto Columns * feat: update tests to add column_info argument at end * feat: remove deepcopy during deserialization of proto message * feat: tests refactoring * feat: integration tests refactoring * feat: samples and sample tests refactoring * feat: lint tests folder * feat:lint samples directory * feat: stop running emulator with proto ddl commands * feat: close the file after reading * feat: update protobuf version lower bound to >3.20 to check proto message compatibility * feat: update setup for snippets_tests.py file * feat: add integration tests * feat: remove duplicate integration tests * feat: add proto_descriptor parameter to required tests * feat: add compatibility tests between Proto message, Bytes and Proto Enum, Int64 * feat: add index tests for proto columns * feat: replace duplicates with sample data * feat: update protobuf lower bound version in setup.py file to add support for proto messages and enum * feat: lint fixes * feat: lint fix * feat: tests refactoring * feat: change comment from dml to dql for read * feat: tests refactoring for update db operation * feat: rever autogenerated code * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: fix code * fix: fix code * fix(spanner): fix code * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix(spanner): skip emulator due to b/338557401 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix(spanner): remove samples * fix(spanner): update coverage * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore(spanner): update coverage * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix(spanner): add samples and update proto schema * fix(spanner): update samples database and emulator DDL * fix(spanner): update admin test to use autogenerated interfaces * fix(spanner): comment refactoring --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent bc71fe9 commit 3ca2689

32 files changed

Lines changed: 1223 additions & 71 deletions

‎google/cloud/spanner_v1/_helpers.py‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@
1818
import decimal
1919
import math
2020
import time
21+
import base64
2122

2223
from google.protobuf.struct_pb2 import ListValue
2324
from google.protobuf.struct_pb2 import Value
25+
from google.protobuf.message import Message
26+
from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
2427

2528
from google.api_core import datetime_helpers
2629
from google.cloud._helpers import _date_from_iso8601_date
@@ -204,6 +207,12 @@ def _make_value_pb(value):
204207
return Value(null_value="NULL_VALUE")
205208
else:
206209
return Value(string_value=value)
210+
if isinstance(value, Message):
211+
value = value.SerializeToString()
212+
if value is None:
213+
return Value(null_value="NULL_VALUE")
214+
else:
215+
return Value(string_value=base64.b64encode(value))
207216

208217
raise ValueError("Unknown type: %s" % (value,))
209218

@@ -232,7 +241,7 @@ def _make_list_value_pbs(values):
232241
return [_make_list_value_pb(row) for row in values]
233242

234243

235-
def _parse_value_pb(value_pb, field_type):
244+
def _parse_value_pb(value_pb, field_type, field_name, column_info=None):
236245
"""Convert a Value protobuf to cell data.
237246
238247
:type value_pb: :class:`~google.protobuf.struct_pb2.Value`
@@ -241,6 +250,18 @@ def _parse_value_pb(value_pb, field_type):
241250
:type field_type: :class:`~google.cloud.spanner_v1.types.Type`
242251
:param field_type: type code for the value
243252
253+
:type field_name: str
254+
:param field_name: column name
255+
256+
:type column_info: dict
257+
:param column_info: (Optional) dict of column name and column information.
258+
An object where column names as keys and custom objects as corresponding
259+
values for deserialization. It's specifically useful for data types like
260+
protobuf where deserialization logic is on user-specific code. When provided,
261+
the custom object enables deserialization of backend-received column data.
262+
If not provided, data remains serialized as bytes for Proto Messages and
263+
integer for Proto Enums.
264+
244265
:rtype: varies on field_type
245266
:returns: value extracted from value_pb
246267
:raises ValueError: if unknown type is passed
@@ -273,18 +294,38 @@ def _parse_value_pb(value_pb, field_type):
273294
return DatetimeWithNanoseconds.from_rfc3339(value_pb.string_value)
274295
elif type_code == TypeCode.ARRAY:
275296
return [
276-
_parse_value_pb(item_pb, field_type.array_element_type)
297+
_parse_value_pb(
298+
item_pb, field_type.array_element_type, field_name, column_info
299+
)
277300
for item_pb in value_pb.list_value.values
278301
]
279302
elif type_code == TypeCode.STRUCT:
280303
return [
281-
_parse_value_pb(item_pb, field_type.struct_type.fields[i].type_)
304+
_parse_value_pb(
305+
item_pb, field_type.struct_type.fields[i].type_, field_name, column_info
306+
)
282307
for (i, item_pb) in enumerate(value_pb.list_value.values)
283308
]
284309
elif type_code == TypeCode.NUMERIC:
285310
return decimal.Decimal(value_pb.string_value)
286311
elif type_code == TypeCode.JSON:
287312
return JsonObject.from_str(value_pb.string_value)
313+
elif type_code == TypeCode.PROTO:
314+
bytes_value = base64.b64decode(value_pb.string_value)
315+
if column_info is not None and column_info.get(field_name) is not None:
316+
default_proto_message = column_info.get(field_name)
317+
if isinstance(default_proto_message, Message):
318+
proto_message = type(default_proto_message)()
319+
proto_message.ParseFromString(bytes_value)
320+
return proto_message
321+
return bytes_value
322+
elif type_code == TypeCode.ENUM:
323+
int_value = int(value_pb.string_value)
324+
if column_info is not None and column_info.get(field_name) is not None:
325+
proto_enum = column_info.get(field_name)
326+
if isinstance(proto_enum, EnumTypeWrapper):
327+
return proto_enum.Name(int_value)
328+
return int_value
288329
else:
289330
raise ValueError("Unknown type: %s" % (field_type,))
290331

@@ -305,7 +346,7 @@ def _parse_list_value_pbs(rows, row_type):
305346
for row in rows:
306347
row_data = []
307348
for value_pb, field in zip(row.values, row_type.fields):
308-
row_data.append(_parse_value_pb(value_pb, field.type_))
349+
row_data.append(_parse_value_pb(value_pb, field.type_, field.name))
309350
result.append(row_data)
310351
return result
311352

‎google/cloud/spanner_v1/data_types.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
"""Custom data types for spanner."""
1616

1717
import json
18+
import types
19+
20+
from google.protobuf.message import Message
21+
from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
1822

1923

2024
class JsonObject(dict):
@@ -71,3 +75,109 @@ def serialize(self):
7175
return json.dumps(self._array_value, sort_keys=True, separators=(",", ":"))
7276

7377
return json.dumps(self, sort_keys=True, separators=(",", ":"))
78+
79+
80+
def _proto_message(bytes_val, proto_message_object):
81+
"""Helper for :func:`get_proto_message`.
82+
parses serialized protocol buffer bytes data into proto message.
83+
84+
Args:
85+
bytes_val (bytes): bytes object.
86+
proto_message_object (Message): Message object for parsing
87+
88+
Returns:
89+
Message: parses serialized protocol buffer data into this message.
90+
91+
Raises:
92+
ValueError: if the input proto_message_object is not of type Message
93+
"""
94+
if isinstance(bytes_val, types.NoneType):
95+
return None
96+
97+
if not isinstance(bytes_val, bytes):
98+
raise ValueError("Expected input bytes_val to be a string")
99+
100+
proto_message = proto_message_object.__deepcopy__()
101+
proto_message.ParseFromString(bytes_val)
102+
return proto_message
103+
104+
105+
def _proto_enum(int_val, proto_enum_object):
106+
"""Helper for :func:`get_proto_enum`.
107+
parses int value into string containing the name of an enum value.
108+
109+
Args:
110+
int_val (int): integer value.
111+
proto_enum_object (EnumTypeWrapper): Enum object.
112+
113+
Returns:
114+
str: string containing the name of an enum value.
115+
116+
Raises:
117+
ValueError: if the input proto_enum_object is not of type EnumTypeWrapper
118+
"""
119+
if isinstance(int_val, types.NoneType):
120+
return None
121+
122+
if not isinstance(int_val, int):
123+
raise ValueError("Expected input int_val to be a integer")
124+
125+
return proto_enum_object.Name(int_val)
126+
127+
128+
def get_proto_message(bytes_string, proto_message_object):
129+
"""parses serialized protocol buffer bytes' data or its list into proto message or list of proto message.
130+
131+
Args:
132+
bytes_string (bytes or list[bytes]): bytes object.
133+
proto_message_object (Message): Message object for parsing
134+
135+
Returns:
136+
Message or list[Message]: parses serialized protocol buffer data into this message.
137+
138+
Raises:
139+
ValueError: if the input proto_message_object is not of type Message
140+
"""
141+
if isinstance(bytes_string, types.NoneType):
142+
return None
143+
144+
if not isinstance(proto_message_object, Message):
145+
raise ValueError("Input proto_message_object should be of type Message")
146+
147+
if not isinstance(bytes_string, (bytes, list)):
148+
raise ValueError(
149+
"Expected input bytes_string to be a string or list of strings"
150+
)
151+
152+
if isinstance(bytes_string, list):
153+
return [_proto_message(item, proto_message_object) for item in bytes_string]
154+
155+
return _proto_message(bytes_string, proto_message_object)
156+
157+
158+
def get_proto_enum(int_value, proto_enum_object):
159+
"""parses int or list of int values into enum or list of enum values.
160+
161+
Args:
162+
int_value (int or list[int]): list of integer value.
163+
proto_enum_object (EnumTypeWrapper): Enum object.
164+
165+
Returns:
166+
str or list[str]: list of strings containing the name of enum value.
167+
168+
Raises:
169+
ValueError: if the input int_list is not of type list
170+
"""
171+
if isinstance(int_value, types.NoneType):
172+
return None
173+
174+
if not isinstance(proto_enum_object, EnumTypeWrapper):
175+
raise ValueError("Input proto_enum_object should be of type EnumTypeWrapper")
176+
177+
if not isinstance(int_value, (int, list)):
178+
raise ValueError("Expected input int_value to be a integer or list of integers")
179+
180+
if isinstance(int_value, list):
181+
return [_proto_enum(item, proto_enum_object) for item in int_value]
182+
183+
return _proto_enum(int_value, proto_enum_object)

‎google/cloud/spanner_v1/database.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,9 @@ class Database(object):
137137
:type enable_drop_protection: boolean
138138
:param enable_drop_protection: (Optional) Represents whether the database
139139
has drop protection enabled or not.
140+
:type proto_descriptors: bytes
141+
:param proto_descriptors: (Optional) Proto descriptors used by CREATE/ALTER PROTO BUNDLE
142+
statements in 'ddl_statements' above.
140143
"""
141144

142145
_spanner_api = None
@@ -152,6 +155,7 @@ def __init__(
152155
database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED,
153156
database_role=None,
154157
enable_drop_protection=False,
158+
proto_descriptors=None,
155159
):
156160
self.database_id = database_id
157161
self._instance = instance
@@ -173,6 +177,7 @@ def __init__(
173177
self._enable_drop_protection = enable_drop_protection
174178
self._reconciling = False
175179
self._directed_read_options = self._instance._client.directed_read_options
180+
self._proto_descriptors = proto_descriptors
176181

177182
if pool is None:
178183
pool = BurstyPool(database_role=database_role)
@@ -382,6 +387,14 @@ def enable_drop_protection(self):
382387
def enable_drop_protection(self, value):
383388
self._enable_drop_protection = value
384389

390+
@property
391+
def proto_descriptors(self):
392+
"""Proto Descriptors for this database.
393+
:rtype: bytes
394+
:returns: bytes representing the proto descriptors for this database
395+
"""
396+
return self._proto_descriptors
397+
385398
@property
386399
def logger(self):
387400
"""Logger used by the database.
@@ -465,6 +478,7 @@ def create(self):
465478
extra_statements=list(self._ddl_statements),
466479
encryption_config=self._encryption_config,
467480
database_dialect=self._database_dialect,
481+
proto_descriptors=self._proto_descriptors,
468482
)
469483
future = api.create_database(request=request, metadata=metadata)
470484
return future
@@ -501,6 +515,7 @@ def reload(self):
501515
metadata = _metadata_with_prefix(self.name)
502516
response = api.get_database_ddl(database=self.name, metadata=metadata)
503517
self._ddl_statements = tuple(response.statements)
518+
self._proto_descriptors = response.proto_descriptors
504519
response = api.get_database(name=self.name, metadata=metadata)
505520
self._state = DatabasePB.State(response.state)
506521
self._create_time = response.create_time
@@ -514,7 +529,7 @@ def reload(self):
514529
self._enable_drop_protection = response.enable_drop_protection
515530
self._reconciling = response.reconciling
516531

517-
def update_ddl(self, ddl_statements, operation_id=""):
532+
def update_ddl(self, ddl_statements, operation_id="", proto_descriptors=None):
518533
"""Update DDL for this database.
519534
520535
Apply any configured schema from :attr:`ddl_statements`.
@@ -526,6 +541,8 @@ def update_ddl(self, ddl_statements, operation_id=""):
526541
:param ddl_statements: a list of DDL statements to use on this database
527542
:type operation_id: str
528543
:param operation_id: (optional) a string ID for the long-running operation
544+
:type proto_descriptors: bytes
545+
:param proto_descriptors: (optional) Proto descriptors used by CREATE/ALTER PROTO BUNDLE statements
529546
530547
:rtype: :class:`google.api_core.operation.Operation`
531548
:returns: an operation instance
@@ -539,6 +556,7 @@ def update_ddl(self, ddl_statements, operation_id=""):
539556
database=self.name,
540557
statements=ddl_statements,
541558
operation_id=operation_id,
559+
proto_descriptors=proto_descriptors,
542560
)
543561

544562
future = api.update_database_ddl(request=request, metadata=metadata)

‎google/cloud/spanner_v1/instance.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,7 @@ def database(
435435
enable_drop_protection=False,
436436
# should be only set for tests if tests want to use interceptors
437437
enable_interceptors_in_tests=False,
438+
proto_descriptors=None,
438439
):
439440
"""Factory to create a database within this instance.
440441
@@ -478,9 +479,14 @@ def database(
478479
:param enable_interceptors_in_tests: (Optional) should only be set to True
479480
for tests if the tests want to use interceptors.
480481
482+
:type proto_descriptors: bytes
483+
:param proto_descriptors: (Optional) Proto descriptors used by CREATE/ALTER PROTO BUNDLE
484+
statements in 'ddl_statements' above.
485+
481486
:rtype: :class:`~google.cloud.spanner_v1.database.Database`
482487
:returns: a database owned by this instance.
483488
"""
489+
484490
if not enable_interceptors_in_tests:
485491
return Database(
486492
database_id,
@@ -492,6 +498,7 @@ def database(
492498
database_dialect=database_dialect,
493499
database_role=database_role,
494500
enable_drop_protection=enable_drop_protection,
501+
proto_descriptors=proto_descriptors,
495502
)
496503
else:
497504
return TestDatabase(

‎google/cloud/spanner_v1/param_types.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
from google.cloud.spanner_v1 import TypeAnnotationCode
1919
from google.cloud.spanner_v1 import TypeCode
2020
from google.cloud.spanner_v1 import StructType
21+
from google.protobuf.message import Message
22+
from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
2123

2224

2325
# Scalar parameter types
@@ -73,3 +75,35 @@ def Struct(fields):
7375
:returns: the appropriate struct-type protobuf
7476
"""
7577
return Type(code=TypeCode.STRUCT, struct_type=StructType(fields=fields))
78+
79+
80+
def ProtoMessage(proto_message_object):
81+
"""Construct a proto message type description protobuf.
82+
83+
:type proto_message_object: :class:`google.protobuf.message.Message`
84+
:param proto_message_object: the proto message instance
85+
86+
:rtype: :class:`type_pb2.Type`
87+
:returns: the appropriate proto-message-type protobuf
88+
"""
89+
if not isinstance(proto_message_object, Message):
90+
raise ValueError("Expected input object of type Proto Message.")
91+
return Type(
92+
code=TypeCode.PROTO, proto_type_fqn=proto_message_object.DESCRIPTOR.full_name
93+
)
94+
95+
96+
def ProtoEnum(proto_enum_object):
97+
"""Construct a proto enum type description protobuf.
98+
99+
:type proto_enum_object: :class:`google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper`
100+
:param proto_enum_object: the proto enum instance
101+
102+
:rtype: :class:`type_pb2.Type`
103+
:returns: the appropriate proto-enum-type protobuf
104+
"""
105+
if not isinstance(proto_enum_object, EnumTypeWrapper):
106+
raise ValueError("Expected input object of type Proto Enum")
107+
return Type(
108+
code=TypeCode.ENUM, proto_type_fqn=proto_enum_object.DESCRIPTOR.full_name
109+
)

0 commit comments

Comments
 (0)