From d6652e92d0dcdecbb6f4afa3e3af2288904afb59 Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Thu, 28 May 2026 18:00:17 +1000 Subject: [PATCH 1/5] fix: add more logs to debug tracing otlp connection issue --- Cargo.lock | 4 +- chronos_bin/Cargo.toml | 2 +- chronos_bin/src/bin/chronos.rs | 1 - chronos_bin/src/telemetry/otlp_collector.rs | 16 +++--- .../src/telemetry/register_telemetry.rs | 49 +++++++++++++------ pg_mig/Cargo.toml | 2 +- 6 files changed, 49 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b3b1b..d55bd29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,7 +226,7 @@ dependencies = [ [[package]] name = "chronos_bin" -version = "0.2.1" +version = "0.2.4" dependencies = [ "anyhow", "async-trait", @@ -1473,7 +1473,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pg_mig" -version = "0.2.1" +version = "0.2.4" dependencies = [ "async-trait", "cargo-husky", diff --git a/chronos_bin/Cargo.toml b/chronos_bin/Cargo.toml index de10f7d..ade86a6 100644 --- a/chronos_bin/Cargo.toml +++ b/chronos_bin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chronos_bin" -version = "0.2.1" +version = "0.2.4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/chronos_bin/src/bin/chronos.rs b/chronos_bin/src/bin/chronos.rs index 3ab10e7..b444640 100644 --- a/chronos_bin/src/bin/chronos.rs +++ b/chronos_bin/src/bin/chronos.rs @@ -11,7 +11,6 @@ use std::time::Duration; #[tokio::main] async fn main() { - env_logger::init(); dotenvy::dotenv().ok(); //registering traces diff --git a/chronos_bin/src/telemetry/otlp_collector.rs b/chronos_bin/src/telemetry/otlp_collector.rs index c7c5c7f..6861d90 100644 --- a/chronos_bin/src/telemetry/otlp_collector.rs +++ b/chronos_bin/src/telemetry/otlp_collector.rs @@ -16,14 +16,16 @@ impl OtlpCollector { .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set"))?; global::set_text_map_propagator(TraceContextPropagator::new()); + log::debug!("[telemetry] setting text map propagator"); let exporter = opentelemetry_otlp::SpanExporter::builder().with_tonic().with_endpoint(endpoint).build()?; - + log::debug!("[telemetry] setting exporter"); let provider = SdkTracerProvider::builder().with_batch_exporter(exporter).build(); - + log::debug!("[telemetry] setting tracer provider"); global::set_tracer_provider(provider.clone()); - + log::debug!("[telemetry] setting tracer provider"); let service_name = std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "chronos".to_string()); + log::debug!("[telemetry] setting service name"); Ok(provider.tracer(service_name)) } @@ -32,14 +34,16 @@ impl OtlpCollector { .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set"))?; global::set_text_map_propagator(TraceContextPropagator::new()); + log::debug!("[telemetry] setting text map propagator"); let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().with_endpoint(endpoint).build()?; - + log::debug!("[telemetry] setting exporter"); let provider = SdkTracerProvider::builder().with_batch_exporter(exporter).build(); - + log::debug!("[telemetry] setting tracer provider"); global::set_tracer_provider(provider.clone()); - + log::debug!("[telemetry] setting tracer provider"); let service_name = std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "chronos".to_string()); + log::debug!("[telemetry] setting service name"); Ok(provider.tracer(service_name)) } } diff --git a/chronos_bin/src/telemetry/register_telemetry.rs b/chronos_bin/src/telemetry/register_telemetry.rs index f1e98c2..4cb68bb 100644 --- a/chronos_bin/src/telemetry/register_telemetry.rs +++ b/chronos_bin/src/telemetry/register_telemetry.rs @@ -1,3 +1,4 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; use super::otlp_collector::OtlpCollector; @@ -24,35 +25,55 @@ impl TelemetryCollector { } pub fn register_traces(self) { - let tracer = match self.collector_type { - TelemetryCollectorType::Otlp => { - let otlp_collector = OtlpCollector::new(); - let protocol = std::env::var("OTEL_EXPORTER_OTLP_PROTOCOL").unwrap_or_else(|_| "grpc".to_string()); - if protocol.to_lowercase().contains("grpc") { - otlp_collector.grpc_collector_connect() - } else { - otlp_collector.http_collector_connect() - } + let protocol = std::env::var("OTEL_EXPORTER_OTLP_PROTOCOL").unwrap_or_else(|_| "grpc".to_string()); + let traces_endpoint = std::env::var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT").unwrap_or_else(|_| "".to_string()); + let service_name = std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "".to_string()); + log::info!( + "[telemetry] starting register_traces collector=Otlp protocol={} endpoint={} service={}", + protocol, + traces_endpoint, + service_name + ); + let tracer_result = catch_unwind(AssertUnwindSafe(|| { + let otlp_collector = OtlpCollector::new(); + if protocol.to_lowercase().contains("grpc") { + log::debug!("[telemetry] using grpc_collector_connect"); + otlp_collector.grpc_collector_connect() + } else { + log::debug!("[telemetry] using http_collector_connect"); + otlp_collector.http_collector_connect() + } + })); + let tracer = match tracer_result { + Ok(result) => result, + Err(panic_payload) => { + log::error!("[telemetry] PANIC while creating tracer: {:?}", panic_payload); + return; // keep app alive, logs still work } }; - match tracer { Ok(tracer) => { + log::debug!("[telemetry] tracer created successfully, initializing subscriber"); let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let init_result = tracing_subscriber::registry() .with(filter) .with(tracing_subscriber::fmt::layer()) .with(otel_layer) .try_init(); - if let Err(e) = init_result { - eprintln!("failed to initialize tracing subscriber: {e}"); + log::error!("[telemetry] subscriber init failed: {e}"); + } else { + log::debug!("[telemetry] subscriber init OK"); } } Err(e) => { - log::error!("error while initializing tracing {}", e); + log::error!("[telemetry] tracer creation returned error: {e}"); + let mut source = e.source(); + while let Some(s) = source { + log::error!("[telemetry] caused by: {s}"); + source = s.source(); + } } } } diff --git a/pg_mig/Cargo.toml b/pg_mig/Cargo.toml index 5374141..383c746 100644 --- a/pg_mig/Cargo.toml +++ b/pg_mig/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pg_mig" -version = "0.2.1" +version = "0.2.4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 897d96fa1619989e0d21e2dc5ffe712086a881cd Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Thu, 28 May 2026 18:20:14 +1000 Subject: [PATCH 2/5] chore: add a test image push workflow --- .github/workflows/release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c26d2a7..2e231c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,6 @@ name: release app binary on tag # and creating a GitHub release # and publish the release to GitHub on: - release: - types: [created] push: tags: # only build on tags that start with 'v' From c268ff9434e4b0cfc4fa070a5f115237f3bad563 Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Fri, 29 May 2026 08:00:06 +1000 Subject: [PATCH 3/5] chore: add tes image publish worklow --- .github/workflows/build.yml | 2 +- .github/workflows/publish-test-image.yml | 54 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish-test-image.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 152c4e0..9341c91 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: release app binary on tag +name: build app binary on tag # push to branch on: diff --git a/.github/workflows/publish-test-image.yml b/.github/workflows/publish-test-image.yml new file mode 100644 index 0000000..31ee203 --- /dev/null +++ b/.github/workflows/publish-test-image.yml @@ -0,0 +1,54 @@ +name: publish test image + +on: + workflow_dispatch: + inputs: + test_tag: + description: "Optional test tag (e.g. test-v0.11.1-rc1)" + required: false + type: string + push: + tags: + - "test-v*" + +jobs: + build-and-publish-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.AMN_PAT }} + + - name: Compute tags + id: vars + run: | + INPUT_TAG="${{ github.event.inputs.test_tag }}" + REF_TAG="${GITHUB_REF_NAME}" + TAG="${INPUT_TAG:-$REF_TAG}" + if [ -z "$TAG" ]; then TAG="test-${GITHUB_SHA::7}"; fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + # echo "sha_tag=test-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + + - name: Build and push chronos test image + run: | + docker build -f Dockerfile.chronos . \ + --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} \ + docker push ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} + + - name: Build and push migration test image + run: | + docker build -f Dockerfile.chronos-pg-migrations . \ + --tag ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} \ + docker push ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} + + # In case sha tag is needed, use the following: + # docker build -f Dockerfile.chronos . \ + # --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} \ + # --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.sha_tag }} + # docker push ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.sha_tag }} \ No newline at end of file From 075b79c4f2add122dfc6a84d6b809b4cec6c9275 Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Fri, 29 May 2026 08:48:46 +1000 Subject: [PATCH 4/5] chore: remove new line come docker command --- .github/workflows/publish-test-image.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-test-image.yml b/.github/workflows/publish-test-image.yml index 31ee203..70f70de 100644 --- a/.github/workflows/publish-test-image.yml +++ b/.github/workflows/publish-test-image.yml @@ -37,15 +37,13 @@ jobs: - name: Build and push chronos test image run: | - docker build -f Dockerfile.chronos . \ - --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} \ + docker build -f Dockerfile.chronos . --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} \ docker push ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} - name: Build and push migration test image run: | - docker build -f Dockerfile.chronos-pg-migrations . \ - --tag ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} \ - docker push ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} + docker build -f Dockerfile.chronos-pg-migrations . --tag ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} \ + docker push ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} # In case sha tag is needed, use the following: # docker build -f Dockerfile.chronos . \ From d030ea7e95008ab69d4642a61b56e659d9e1949a Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Fri, 29 May 2026 09:19:18 +1000 Subject: [PATCH 5/5] chore: remove new line come docker command --- .github/workflows/publish-test-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-test-image.yml b/.github/workflows/publish-test-image.yml index 70f70de..f687b57 100644 --- a/.github/workflows/publish-test-image.yml +++ b/.github/workflows/publish-test-image.yml @@ -37,12 +37,12 @@ jobs: - name: Build and push chronos test image run: | - docker build -f Dockerfile.chronos . --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} \ + docker build -f Dockerfile.chronos . --tag ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} docker push ghcr.io/${{ github.repository }}:${{ steps.vars.outputs.tag }} - name: Build and push migration test image run: | - docker build -f Dockerfile.chronos-pg-migrations . --tag ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} \ + docker build -f Dockerfile.chronos-pg-migrations . --tag ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} docker push ghcr.io/${{ github.repository }}/db-migration:${{ steps.vars.outputs.tag }} # In case sha tag is needed, use the following: