{ "schema_version": "0.6.0", "paper_id": "combining-distributed-kernel-tracing-cloud", "page_url": "https://naser.github.io/research-publications/papers/combining-distributed-kernel-tracing-cloud/", "title": "Combining Distributed and Kernel Tracing for Performance Analysis of Cloud Applications", "title_variants": [], "authors": [ "Loïc Gelle", "Naser Ezzati-Jivan", "Michel R. Dagenais" ], "author_details": [ { "name": "Loïc Gelle", "orcid": null, "profile_url": null }, { "name": "Naser Ezzati-Jivan", "orcid": "https://orcid.org/0000-0003-1435-6297", "profile_url": "https://naser.github.io/" }, { "name": "Michel R. Dagenais", "orcid": null, "profile_url": null } ], "publication": { "year": 2021, "venue": "Electronics", "type": "journal article", "publication_date": "2021-10-26", "online_date": "2021-10-26", "print_date": null, "volume": "10", "issue": "21", "pages": "2610", "article_number": null, "publisher": "MDPI AG", "issn": [ "2079-9292" ], "isbn": [], "crossref_type": "journal-article" }, "publication_type": "research paper", "status": "published_with_public_full_text", "canonical_source_url": "https://doi.org/10.3390/electronics10212610", "source_record_id": "combining-distributed-and-kernel-tracing-for-performance-analysis-of-cloud-applications-4453f8bab8", "identifiers": { "doi": "10.3390/electronics10212610" }, "abstract": "Distributed tracing allows tracking user requests that span across multiple services and machines in a distributed application. However, typical cloud applications rely on abstraction layers that can hide the root cause of latency happening between processes or in the kernel. Because of its focus on high-level events, existing methodologies in applying distributed tracing can be limited when trying to detect complex contentions and relate them back to the originating requests. Cross-level analyses that include kernel-level events are necessary to debug problems as prevalent as mutex or disk contention, however cross-level analysis and associating events in the kernel and distributed tracing data is complex and can add a lot of overhead. This paper describes a new solution for combining distributed tracing with low-level software tracing in order to find the latency root cause better. We explain how we achieve a hybrid trace collection to capture and synchronize both kernel and distributed request events. Then, we present our design and implementation for a critical path analysis. We show that our analysis describes precisely how each request spends its time and what stands in its critical path while limiting overhead.", "abstract_source": "Official MDPI full-text HTML, CC BY 4.0, reviewed 2026-08-06: https://www.mdpi.com/2079-9292/10/21/2610", "abstract_available": true, "scholar_eligibility": { "eligible": false, "basis": "not-eligible", "note": "The page is a discovery record; it does not claim Google Scholar article-host eligibility." }, "description": "Distributed tracing allows tracking user requests that span across multiple services and machines in a distributed application. However, typical cloud applications rely on abstraction layers that can hide the root cause of latency happening between processes or in the kernel. Because of its focus on high-level events, existing methodologies in applying distributed tracing can be limited when trying to detect complex contentions and relate them back to the originating requests. Cross-level analyses that include kernel-level events are necessary to debug problems as prevalent as mutex or disk contention, however cross-level analysis and associating events in the kernel and distributed tracing data is complex and can add a lot of overhead. This paper describes a new solution for combining distributed tracing with low-level software tracing in order to find the latency root cause better. We explain how we achieve a hybrid trace collection to capture and synchronize both kernel and distributed request events. Then, we present our design and implementation for a critical path analysis. We show that our analysis describes precisely how each request spends its time and what stands in its critical path while limiting overhead.", "evidence_level": "full-text-reviewed", "evidence": { "source_basis": "full-text-reviewed", "coverage": "material paper sections", "summary_origin": "AI-assisted catalog editorial summary", "review_status": "catalog-reviewed; paper-author approval pending", "verified_on": "2026-08-09", "sources": [ { "note": "Crossref and local DBLP/venue metadata for 10.3390/electronics10212610" }, { "note": "author identity matched to Naser Ezzati-Jivan in the local research catalog" }, { "note": "Official MDPI HTML, Abstract and Sections 1-3 (reviewed 2026-08-06): distributed/kernel tracing motivation, LTTng/Jaeger/Trace Compass stack, patched Java and Go clients, synchronization events, and request critical-path construction." }, { "note": "Official MDPI HTML, Section 4 and Sections 4.1-4.1.2: i7-7820X/32 GB environment, HotROD 10,000-request evaluation, Cassandra 1,000,000-read/write evaluation, sampling configurations, and overhead bounds." }, { "note": "Official MDPI HTML, Sections 4.2-4.4: approximately 15-second analysis for 500 MB/20-30-second traces, 1% CPU-control-group use case, 2-second versus approximately 5-millisecond request latency, recurring 100-ms preemption, and transferability limits." }, { "note": "Official MDPI HTML, Section 5 and Data Availability Statement: dashboard-triggered snapshots, user-level scheduler/thread-pool tracking, network analysis, multi-host extension, CC BY 4.0 license, and https://github.com/loicgelle/jaeger-go-lttng-instr." }, { "note": "The official MDPI PDF URL was verified from the article's citation_pdf_url metadata, but Chrome's PDF endpoint was blocked by an interstitial in this run; no local PDF hash is claimed." } ] }, "summary": { "core_contribution": "The paper combines Jaeger distributed traces with LTTng kernel and user-space events and extends critical-path analysis from threads to distributed requests so request latency can be related to low-level scheduling and resource behavior.", "problem": "Distributed request spans identify application-level paths but can miss kernel scheduling, I/O, and CPU contention that explains hidden latency.", "method": "Synchronize Jaeger spans with LTTng user-space/kernel events, decode CTF with Babeltrace, and analyze request critical paths in Trace Compass with cross-layer scheduling/resource states.", "findings": "HotROD and Cassandra experiments show modest tracing overhead under the stated loads. The combined analysis identifies CPU wait/preemption in a Cassandra CPU-control-group case that Jaeger-only tracing cannot explain; a 500 MB trace can be analyzed in about 15 s.", "limitations": "Evidence is remote HTML only; local PDF pages, local PDF hash, and visual PDF checks are unavailable. The evaluated services/workloads are HotROD and Cassandra, and the paper's future work identifies dashboard-triggered snapshots, work-queue/thread-pool tracking, network analysis, and multi-host extension as open areas.", "future_work": "Trigger snapshots from a dashboard, improve tracking of user schedulers/work queues/thread pools, add network analysis, and extend the method across multiple hosts." }, "tags": [ "kernel-tracing", "system-tracing", "performance-analysis", "observability", "lttng" ], "keywords": [ "distributed tracing", "kernel tracing", "cloud applications", "cross-layer performance analysis", "LTTng", "Jaeger", "Trace Compass", "Babeltrace", "Common Trace Format", "HotROD", "Cassandra", "Apache ab", "cassandra-stress", "critical-path analysis", "snapshot tracing", "CPU control groups", "liblttng-ust", "request synchronization" ], "versions": [ { "id": "published-version", "label": "Published version", "relation": "version-of-record", "title": "Combining Distributed and Kernel Tracing for Performance Analysis of Cloud Applications", "url": "https://doi.org/10.3390/electronics10212610", "pdf_url": null, "status": "published", "canonical_for_citation": true }, { "id": "public-full-text", "label": "Public full text", "relation": "source-record", "title": "Combining Distributed and Kernel Tracing for Performance Analysis of Cloud Applications", "url": "https://www.mdpi.com/2079-9292/10/21/2610/pdf?version=1635235463", "pdf_url": "https://www.mdpi.com/2079-9292/10/21/2610/pdf?version=1635235463", "status": "public_full_text", "canonical_for_citation": false } ], "access": { "status": "published_with_public_full_text", "note": "The MDPI article and PDF are externally linked under the article's CC BY 4.0 license; the PDF is not redistributed in this repository.", "license": "CC-BY-4.0" }, "resources": { "code": "https://github.com/loicgelle/jaeger-go-lttng-instr", "data": null, "slides": null, "demo": null }, "citation_guidance": { "when_to_cite": "Cite this paper when your work uses or compares synchronizing Jaeger spans with LTTng tracepoints carrying request IDs.", "points": [ "For synchronizing Jaeger spans with LTTng tracepoints carrying request IDs.", "For extending thread critical paths into request critical paths with scheduler, I/O, and wait states.", "For the HotROD/Cassandra overhead measurements and 500 MB/15 s analysis result.", "For identifying CPU-control-group preemption/CPU wait as a distributed-service latency cause." ], "canonical_version_id": "published-version" }, "provenance": { "metadata_verified_on": "2026-08-09", "metadata_source": [ "Crossref and local DBLP/venue metadata for 10.3390/electronics10212610", "author identity matched to Naser Ezzati-Jivan in the local research catalog", "Official MDPI HTML, Abstract and Sections 1-3 (reviewed 2026-08-06): distributed/kernel tracing motivation, LTTng/Jaeger/Trace Compass stack, patched Java and Go clients, synchronization events, and request critical-path construction.", "Official MDPI HTML, Section 4 and Sections 4.1-4.1.2: i7-7820X/32 GB environment, HotROD 10,000-request evaluation, Cassandra 1,000,000-read/write evaluation, sampling configurations, and overhead bounds.", "Official MDPI HTML, Sections 4.2-4.4: approximately 15-second analysis for 500 MB/20-30-second traces, 1% CPU-control-group use case, 2-second versus approximately 5-millisecond request latency, recurring 100-ms preemption, and transferability limits.", "Official MDPI HTML, Section 5 and Data Availability Statement: dashboard-triggered snapshots, user-level scheduler/thread-pool tracking, network analysis, multi-host extension, CC BY 4.0 license, and https://github.com/loicgelle/jaeger-go-lttng-instr.", "The official MDPI PDF URL was verified from the article's citation_pdf_url metadata, but Chrome's PDF endpoint was blocked by an interstitial in this run; no local PDF hash is claimed." ], "summary_written_by": "AI-assisted", "summary_verified_by": "full-text-grounded catalog review; author approval pending", "linked_preprint_record": null, "author_order_note": null }, "batch": { "phase": 2, "batch_label": "expanded forty-paper release", "status": "included_in_expanded_catalog", "selected_at": "2026-08-09" } }