I was looking at a code base (GCP SDK's monitoring API) trying to drill down to get familiar with some methods, but I hit a wall here: https://cloud.google.com/monitoring/custom-metrics/creating-metrics#monitoring_create_metric-python
Specifically this line descriptor = ga_metric.MetricDescriptor(). How does MetricDescriptor() get generated?
According to comments in metric_pb2 (ga_metric is an alias to it) that file was generated by protobuf. In that module file I see no definition for MetricDescriptor() though. How am I able to call ga_metric.MetricDescriptor()? What part of the code here is generating the MetricDescriptor() method that I'm able to call?
# metric_pb2.py
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.api import label_pb2 as google_dot_api_dot_label__pb2
from google.api import launch_stage_pb2 as google_dot_api_dot_launch__stage__pb2
from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x17google/api/metric.proto\x12\ngoogle.api\x1a\x16google/api/label.proto\x1a\x1dgoogle/api/launch_stage.proto\x1a\x1egoogle/protobuf/duration.proto"\x9f\x06\n\x10MetricDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x08 \x01(\t\x12+\n\x06labels\x18\x02 \x03(\x0b\x32\x1b.google.api.LabelDescriptor\x12<\n\x0bmetric_kind\x18\x03 \x01(\x0e\x32\'.google.api.MetricDescriptor.MetricKind\x12:\n\nvalue_type\x18\x04 \x01(\x0e\x32&.google.api.MetricDescriptor.ValueType\x12\x0c\n\x04unit\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x07 \x01(\t\x12G\n\x08metadata\x18\n \x01(\x0b\x32\x35.google.api.MetricDescriptor.MetricDescriptorMetadata\x12-\n\x0claunch_stage\x18\x0c \x01(\x0e\x32\x17.google.api.LaunchStage\x12 \n\x18monitored_resource_types\x18\r \x03(\t\x1a\xb0\x01\n\x18MetricDescriptorMetadata\x12\x31\n\x0claunch_stage\x18\x01 \x01(\x0e\x32\x17.google.api.LaunchStageB\x02\x18\x01\x12\x30\n\rsample_period\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cingest_delay\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration"O\n\nMetricKind\x12\x1b\n\x17METRIC_KIND_UNSPECIFIED\x10\x00\x12\t\n\x05GAUGE\x10\x01\x12\t\n\x05\x44\x45LTA\x10\x02\x12\x0e\n\nCUMULATIVE\x10\x03"q\n\tValueType\x12\x1a\n\x16VALUE_TYPE_UNSPECIFIED\x10\x00\x12\x08\n\x04\x42OOL\x10\x01\x12\t\n\x05INT64\x10\x02\x12\n\n\x06\x44OUBLE\x10\x03\x12\n\n\x06STRING\x10\x04\x12\x10\n\x0c\x44ISTRIBUTION\x10\x05\x12\t\n\x05MONEY\x10\x06"u\n\x06Metric\x12\x0c\n\x04type\x18\x03 \x01(\t\x12.\n\x06labels\x18\x02 \x03(\x0b\x32\x1e.google.api.Metric.LabelsEntry\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42_\n\x0e\x63om.google.apiB\x0bMetricProtoP\x01Z7google.golang.org/genproto/googleapis/api/metric;metric\xa2\x02\x04GAPIb\x06proto3'
)
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "google.api.metric_pb2", _globals)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b"\n\016com.google.apiB\013MetricProtoP\001Z7google.golang.org/genproto/googleapis/api/metric;metric\242\002\004GAPI"
_METRICDESCRIPTOR_METRICDESCRIPTORMETADATA.fields_by_name[
"launch_stage"
]._options = None
_METRICDESCRIPTOR_METRICDESCRIPTORMETADATA.fields_by_name[
"launch_stage"
]._serialized_options = b"\030\001"
_METRIC_LABELSENTRY._options = None
_METRIC_LABELSENTRY._serialized_options = b"8\001"
_globals["_METRICDESCRIPTOR"]._serialized_start = 127
_globals["_METRICDESCRIPTOR"]._serialized_end = 926
_globals["_METRICDESCRIPTOR_METRICDESCRIPTORMETADATA"]._serialized_start = 554
_globals["_METRICDESCRIPTOR_METRICDESCRIPTORMETADATA"]._serialized_end = 730
_globals["_METRICDESCRIPTOR_METRICKIND"]._serialized_start = 732
_globals["_METRICDESCRIPTOR_METRICKIND"]._serialized_end = 811
_globals["_METRICDESCRIPTOR_VALUETYPE"]._serialized_start = 813
_globals["_METRICDESCRIPTOR_VALUETYPE"]._serialized_end = 926
_globals["_METRIC"]._serialized_start = 928
_globals["_METRIC"]._serialized_end = 1045
_globals["_METRIC_LABELSENTRY"]._serialized_start = 1000
_globals["_METRIC_LABELSENTRY"]._serialized_end = 1045
# @@protoc_insertion_point(module_scope)
Per DazWilkin I was able to locate all the packages with proto files and generate pyi files for them. This works reasonably well. pylance finds them (though pylint does not?). Also, there is some gnarly bug with protoc and I had to run it grpc_tools to get it to work.
packages_paths = site.getsitepackages()[0]
proto_folders: list[str] = []
for name in glob.glob(f"{packages_paths}/**/*.proto", recursive=True):
proto_folder = os.path.dirname(name)
proto_folders.append(proto_folder)
proto_folders = list(set(proto_folders))
for proto_folder in proto_folders:
os.chdir(proto_folder)
# If we wildcard and there is a single failure the rest are skipped
# So just loop over each file and run protoc for each one
for proto_file in glob.glob(f"{proto_folder}/*.proto", recursive=True):
file_name = os.path.basename(proto_file)
cmd = f"python -m grpc_tools.protoc --proto_path=. --pyi_out=. {file_name}"
< RUN CMD >
See these three lines:
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "google.api.metric_pb2", _globals)
globals() returns the dict implementing the namespace of the current module, so at the module scope the following are equivalent:
foo = "asdf"
# equivalent:
globals()["foo"] = "asdf"
By passing the return value of globals() to the two _builder.Build... functions, the code is allowing the builder to add names to the current module. That's how the builder.Build... functions can define names like MetricDescriptor.
Getting auto-complete to work despite the shenanigans is a tall order. When working with protobufs I like to view the .proto file from which the _pb2.py file is generated (probably metric.proto would generate metric_pb2.py), alongside Google's Python Generated Code Guide.