I'm trying to use OpenTelemetry to export metrics to a remote endpoint. As a test, I'm trying to export a counter to a Prometheus server at localhost:8000. However, when I run the below code, I can't see the metrics at localhost:8000/metrics. Could anyone point me to where I'm getting this wrong? Note that I am using opentelemetry-exporter-prometheus-remote-write to try to accomplish this.
from prometheus_client import start_http_server
import random
import time
from opentelemetry import metrics
from opentelemetry.exporter.prometheus_remote_write import PrometheusRemoteWriteMetricsExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
start_http_server(8000, addr='localhost')
exporter = PrometheusRemoteWriteMetricsExporter(endpoint='http://localhost:8000', headers={})
reader = PeriodicExportingMetricReader(exporter, 1000)
provider = MeterProvider(metric_readers=[reader])
metrics.set_meter_provider(provider)
meter = metrics.get_meter(__name__)
requests_counter = meter.create_counter('test_counter', 'Random counter')
while True:
num = random.randint(0, 50)
print(num)
requests_counter.add(num, {'env': 'staging', 'label': '9999'})
time.sleep(1)
EDIT: Here's some revised code after running
docker run \
[prom image] \
--config.file=/etc/prometheus/prometheus.yml \
--enable-feature=remote-writer-receiver
Unfortunately still not working.
from prometheus_client import start_http_server
import random
import time
from opentelemetry import metrics
from opentelemetry.exporter.prometheus_remote_write import PrometheusRemoteWriteMetricsExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
# start_http_server(8000, addr='localhost')
exporter = PrometheusRemoteWriteMetricsExporter(endpoint='http://localhost:9090/api/v1/write', headers={})
reader = PeriodicExportingMetricReader(exporter, 1000)
provider = MeterProvider(metric_readers=[reader])
metrics.set_meter_provider(provider)
meter = metrics.get_meter('my_meter')
requests_counter = meter.create_counter(name='test_counter', description='Random counter', unit='1')
while True:
num = random.randint(0, 50)
print(num)
requests_counter.add(num)
time.sleep(1)
A couple of things (see Storage: Overview):
/api/v1/write
NB You may want to call
meter.create_counter(
name='test_counter',
description='Random counter',
)
podman run \
--interactive --tty --rm \
--publish=9090:9090/tcp \
docker.io/prom/prometheus:v2.54.1 \
--config.file=/etc/prometheus/prometheus.yml \
--enable-feature=remote-write-receiver
NB The default Prometheus Web endpoint port is 9090
.
Then, after running your (revised) code, I can query the Prometheus API:
curl \
--silent \
--get \
--data-urlencode "query=test_counter[15s]" \
http://localhost:9090/api/v1/query \
| jq -r .
{
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"__name__": "test_counter",
"env": "staging",
"label": "9999",
"service_name": "unknown_service",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.27.0"
},
"values": [
[
1726765795.586,
"344"
],
[
1726765796.598,
"374"
],
[
1726765797.605,
"383"
],
[
1726765798.611,
"409"
],
[
1726765799.616,
"419"
],
[
1726765800.623,
"460"
],
[
1726765801.633,
"476"
],
[
1726765802.641,
"502"
],
[
1726765803.648,
"523"
],
[
1726765804.652,
"548"
],
[
1726765805.66,
"592"
],
[
1726765806.666,
"619"
],
[
1726765807.672,
"668"
],
[
1726765808.679,
"674"
],
[
1726765809.687,
"709"
]
]
}
]
}
}