How to Build a Logging Pipeline with OpenTelemetry, Grafana Loki, and Grafana in Docker Compose

services: # I use .NET 8 to develop the application dotnet-app: build: context: . dockerfile: Dockerfile # Remeber to add Dockerfile of the application container_name: dotnet-app environment: ASPNETCORE_URLS: http://+:5000 DOTNET_LOG_LEVEL: Information ports: - "${DOTNET_APP_PORT}:5000" # .NET App port depends_on: - opentelemetry-collector-svc # Loki (Log storage) loki-svc: image: grafana/loki:3.3.2 container_name: loki-svc ports: - "3100" # Loki API port command: -config.expand-env=true -config.file=/etc/loki/local-config.yaml environment: MINIO_USER: ${MINIO_USER} MINIO_PASSWORD: ${MINIO_PASSWORD} MINIO_PORT: ${MINIO_PORT} volumes: - ./loki-config.yaml:/etc/loki/local-config.yaml depends_on: - minio-svc postgres-grafana-svc: image: postgres container_name: postgres-grafana-svc # set shared memory limit when using docker-compose shm_size: 128mb environment: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_GRAFANA_DB} ports: - "${POSTGRES_GRAFANA_PORT}:5432" volumes: - postgres_grafana_data:/var/lib/postgresql/data # Grafana (Visualization) grafana-svc: image: grafana/grafana container_name: grafana-svc ports: - "${GRAFANA_PORT}:3000" # Grafana UI port environment: GF_DATABASE_TYPE: postgres GF_DATABASE_HOST: postgres-grafana-svc:5432 GF_DATABASE_NAME: ${POSTGRES_GRAFANA_DB} GF_DATABASE_USER: ${GF_SECURITY_ADMIN_USER} GF_DATABASE_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD} depends_on: - postgres-grafana-svc - loki-svc entrypoint: - sh - -euc - | mkdir -p /etc/grafana/provisioning/datasources cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml apiVersion: 1 datasources: - name: Loki type: loki access: proxy orgId: 1 url: http://loki-svc:3100 basicAuth: false isDefault: true version: 1 editable: false EOF /run.sh # OpenTelemetry Collector opentelemetry-collector-svc: image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest container_name: opentelemetry-collector-svc ports: - "4318" # OTLP HTTP port volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml environment: OTEL_LOG_LEVEL: debug command: - "--config=/etc/otel-collector-config.yaml" depends_on: - loki-svc # MiniO (Object Storage) minio-svc: image: minio/minio container_name: minio-svc ports: - "${MINIO_PORT}:9000" # MiniO API port - "${MINIO_CONSOLE_PORT}:9001" # MiniO Console port for web ui environment: MINIO_ROOT_USER: ${MINIO_USER} MINIO_ROOT_PASSWORD: ${MINIO_PASSWORD} command: server /data --console-address ":9001" # 9001 container port volumes: - minio_data:/data volumes: minio_data: postgres_grafana_data: 

POSTGRES_USER=admin POSTGRES_PASSWORD=admin POSTGRES_DB=app POSTGRES_PORT=15432 POSTGRES_GRAFANA_DB=grafana POSTGRES_GRAFANA_PORT=15433 DOTNET_APP_PORT=15000 GRAFANA_PORT=15300 GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=admin MINIO_PORT=15900 MINIO_CONSOLE_PORT=15901 MINIO_USER=minioadmin MINIO_PASSWORD=minioadmin 

receivers: otlp: protocols: http: endpoint: 0.0.0.0:4318 processors: batch: exporters: otlphttp/loki: endpoint: http://loki-svc:3100/otlp tls: insecure: true debug: service: pipelines: logs: receivers: [otlp] processors: [batch] exporters: [otlphttp/loki, debug] 

auth_enabled: false server: http_listen_port: 3100 http_server_write_timeout: 310s http_server_read_timeout: 310s common: path_prefix: /loki storage: s3: endpoint: minio-svc:9000 bucketnames: loki-bucket # important !! insecure: true access_key_id: ${MINIO_USER} secret_access_key: ${MINIO_PASSWORD} s3forcepathstyle: true # important !! replication_factor: 1 # important !! ring: kvstore: store: inmemory ingester: chunk_encoding: snappy chunk_idle_period: 2h chunk_target_size: 1536000 max_chunk_age: 2h querier: max_concurrent: 8 schema_config: configs: - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 index: prefix: index_ period: 24h compactor: working_directory: /loki/compactor limits_config: max_query_parallelism: 24 split_queries_by_interval: 15m ingestion_rate_mb: 20 ingestion_burst_size_mb: 30 per_stream_rate_limit: "3MB" per_stream_rate_limit_burst: "10MB" query_timeout: 300s allow_structured_metadata: true ruler: storage: type: s3 s3: endpoint: minio-svc:9000 bucketnames: loki-rules # important !! insecure: true access_key_id: ${MINIO_USER} secret_access_key: ${MINIO_PASSWORD} s3forcepathstyle: true # important !! 

OpenTelemetry OpenTelemetry.Exporter.OpenTelemetryProtocol OpenTelemetry.Extensions.Hosting 

builder.Logging.ClearProviders().AddConsole().AddOpenTelemetry(options => { options.IncludeScopes = true; options.IncludeFormattedMessage = true; options.SetResourceBuilder(ResourceBuilder.CreateDefault() .AddService(serviceName: "dotnet-app") // your service name .AddAttributes(new Dictionary<string, object> { // add whatever you want ["environment.name"] = "dev", })); options.AddOtlpExporter(otlpOptions => { otlpOptions.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; otlpOptions.Endpoint = new Uri("http://opentelemetry-collector-svc:4318/v1/logs"); // This URL is only for logs, not trace or metric }); }); 

 docker-compose --env-file docker-compose-config.txt up --build -d 

docker-compose down -v 

caller=ratestore.go:109 msg="error getting ingester clients" err="empty ring"