blob: 4b980284e0652e937a73407061a30f4aabaaaa11 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Diagnosing Errors &mdash; Apache Mynewt latest documentation</title>
<link rel="shortcut icon" href="../../_static/mynewt-logo-only-newt32x32.png"/>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/sphinx_theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/bootstrap-3.0.3.min.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/v2.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/custom.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/restructuredtext.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/overrides.css" type="text/css" />
<link rel="index" title="Index"
href="../../genindex.html"/>
<link rel="search" title="Search" href="../../search.html"/>
<link rel="top" title="Apache Mynewt latest documentation" href="../../index.html"/>
<link rel="up" title="Tooling" href="tooling.html"/>
<link rel="next" title="Other" href="../other/other.html"/>
<link rel="prev" title="SEGGER SystemView" href="segger_sysview.html"/>
<script src="../../_static/js/modernizr.min.js"></script>
<script>
(function(i, s, o, g, r, a, m) {
i["GoogleAnalyticsObject"] = r;
(i[r] =
i[r] ||
function() {
(i[r].q = i[r].q || []).push(arguments);
}),
(i[r].l = 1 * new Date());
(a = s.createElement(o)), (m = s.getElementsByTagName(o)[0]);
a.async = 1;
a.src = g;
m.parentNode.insertBefore(a, m);
})(window, document, "script", "//www.google-analytics.com/analytics.js", "ga");
ga("create", "UA-72162311-1", "auto");
ga("send", "pageview");
</script>
</head>
<body class="not-front page-documentation" role="document" >
<div id="wrapper">
<div class="container">
<div id="banner" class="row v2-main-banner">
<a class="logo-cell" href="/">
<img class="logo" src="../../_static/img/logo.png">
</a>
<div class="tagline-cell">
<h4 class="tagline">An OS to build, deploy and securely manage billions of devices</h4>
</div>
<div class="news-cell">
<div class="well">
<h4>Latest News:</h4> <a href="/download">Apache Mynewt 1.11.0, Apache NimBLE 1.6.0 </a> released September 7, 2023)
</div>
</div>
</div>
</div>
<header>
<nav id="navbar" class="navbar navbar-inverse" role="navigation">
<div class="container">
<!-- Collapsed navigation -->
<div class="navbar-header">
<!-- Expander button -->
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<!-- Expanded navigation -->
<div class="navbar-collapse collapse">
<!-- Main navigation -->
<ul class="nav navbar-nav navbar-right">
<li>
<a href="/"><i class="fa fa-home" style="font-size: larger;"></i></a>
</li>
<li class="important">
<a href="/quick-start/">Quick Start</a>
</li>
<li>
<a href="/about/">About</a>
</li>
<li>
<a href="/talks/">Talks</a>
</li>
<li class="active">
<a href="/documentation/">Documentation</a>
</li>
<li>
<a href="/download/">Download</a>
</li>
<li>
<a href="/community/">Community</a>
</li>
<li>
<a href="/events/">Events</a>
</li>
</ul>
<!-- Search, Navigation and Repo links -->
<ul class="nav navbar-nav navbar-right">
</ul>
</div>
</div>
</nav>
</header>
<!-- STARTS MAIN CONTENT -->
<div id="main-content">
<div id="breadcrumb">
<div class="container">
<a href="/documentation/">Docs</a> /
<a href="../tutorials.html">Tutorials</a> /
<a href="tooling.html">Tooling</a> /
Diagnosing Errors
<div class="sourcelink">
<a href="https://github.com/apache/mynewt-documentation/edit/master/docs/tutorials/tooling/error_diagnostics.rst" class="icon icon-github"
rel="nofollow"> Edit on GitHub</a>
</div>
</div>
</div>
<!-- STARTS CONTAINER -->
<div class="container">
<!-- STARTS .content -->
<div id="content" class="row">
<!-- STARTS .container-sidebar -->
<div class="container-sidebar col-xs-12 col-sm-3">
<div id="docSidebar" class="sticky-container">
<div role="search" class="sphinx-search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search documentation" class="search-documentation" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
<!-- Note: only works when deployed -->
<select class="form-control" onchange="if (this.value) window.location.href=this.value">
<option value="/latest" selected>
Version: latest
</option>
<option value="/v1_11_0" >
Version: 1.11.0
</option>
<option value="/v1_10_0" >
Version: 1.10.0
</option>
<option value="/v1_9_0" selected="selected" >
Version: 1.9.0
</option>
<option value="/v1_8_0" >
Version: 1.8.0
</option>
<option value="/v1_7_0" >
Version: 1.7.0
</option>
<option value="/v1_6_0" >
Version: 1.6.0
</option>
<option value="/v1_5_0" >
Version: 1.5.0
</option>
<option value="/v1_4_0" >
Version: 1.4.0
</option>
<option value="/v1_3_0/os/introduction" >
Version: 1.3.0
</option>
<option value="/v1_2_0/os/introduction" >
Version: 1.2.0
</option>
<option value="/v1_1_0/os/introduction" >
Version: 1.1.0
</option>
<option value="/v1_0_0/os/introduction" >
Version: 1.0.0
</option>
<option value="/v0_9_0/os/introduction" >
Version: 0.9.0
</option>
</select>
<div class="region region-sidebar">
<div class="docs-menu">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../index.html">Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../get_started/index.html">Setup &amp; Get Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../tutorials.html">Tutorials</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../blinky/blinky.html">Project Blinky</a></li>
<li class="toctree-l2"><a class="reference internal" href="../repo/add_repos.html">Working with repositories</a></li>
<li class="toctree-l2"><a class="reference internal" href="../slinky/project-slinky.html">Project Slinky for Remote Comms</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ble/ble.html">Bluetooth Low Energy</a></li>
<li class="toctree-l2"><a class="reference internal" href="../lora/lorawanapp.html">LoRa</a></li>
<li class="toctree-l2"><a class="reference internal" href="../os_fundamentals/os_fundamentals.html">OS Fundamentals</a></li>
<li class="toctree-l2"><a class="reference internal" href="../devmgmt/devmgmt.html">Remote Device Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../sensors/sensors.html">Sensors</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="tooling.html">Tooling</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="segger_rtt.html">Segger RTT</a></li>
<li class="toctree-l3"><a class="reference internal" href="segger_sysview.html">Segger Sysview</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Error Diagnostics</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../other/other.html">Other</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../external_links.html">Third-party Resources</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../os/os_user_guide.html">OS User Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../network/index.html">BLE User Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../newt/index.html">Newt Tool Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../newtmgr/index.html">Newt Manager Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mynewt_faq/index.html">Mynewt FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../misc/index.html">Appendix</a></li>
</ul>
</div>
</div>
</div>
<!-- ENDS STICKY CONTAINER -->
</div>
<!-- ENDS .container-sidebar -->
<div class="col-xs-12 col-sm-9">
<div class="alert alert-warning">
<p>
Version 1.9.0 is not the most recent version of the
Apache Mynewt documentation. Click <a href="/latest">here</a> to
read the latest version.
</p>
</div>
<div class="">
<div class="rst-content">
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="diagnosing-errors">
<h1>Diagnosing Errors<a class="headerlink" href="#diagnosing-errors" title="Permalink to this headline"></a></h1>
<p>Here we list some of the tools Mynewt has available. This is by no means
comprehensive. It does give an overview of some facilities more commonly
used.</p>
<p>To create output for this demo, I’m including the ‘test/crash_test’ package
while building the ‘slinky’ app.</p>
<div class="section" id="crash-with-console">
<h2>Crash with Console<a class="headerlink" href="#crash-with-console" title="Permalink to this headline"></a></h2>
<p>When system restarts due to an error, we always attempt to do it
in a way which allows us to dump the current system state. Calls to
assert(), error interrupts (regular faults, memory faults) both take
this path.</p>
<p>The basic facility is the printout of registers to console.</p>
<p>Here is an example of what it looks like with Cortex-M4:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">041088 Unhandled interrupt (3), exception sp 0x20001d28</span>
<span class="go">041088 r0:0x00000000 r1:0x00017b61 r2:0x00000000 r3:0x0000002a</span>
<span class="go">041088 r4:0x20001dc8 r5:0x00000000 r6:0x200034e4 r7:0x20003464</span>
<span class="go">041088 r8:0x2000349c r9:0x20001f08 r10:0x00017bd4 r11:0x00000000</span>
<span class="go">041088 r12:0x00000000 lr:0x00014e29 pc:0x00014e58 psr:0x61000000</span>
<span class="go">041088 ICSR:0x00421803 HFSR:0x40000000 CFSR:0x02000000</span>
<span class="go">041088 BFAR:0xe000ed38 MMFAR:0xe000ed34</span>
</pre></div>
</div>
<p>Output includes the values of registers. The most interesting pieces
would be $pc (program counter), and $lr (link register), which show
the instruction where the fault happened, and the return address from
that function.</p>
<p>You would then take these values, and match them against the image
you’re running on the target.</p>
<p>For example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>arm-elf-linux-gdb<span class="w"> </span>bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf
<span class="go">GNU gdb (GDB) 7.8.1</span>
<span class="go"> ...</span>
<span class="go">Reading symbols from bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf...done.</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00014e58</span>
<span class="go">0x14e58 is in crash_device (repos/apache-mynewt-core/test/crash_test/src/crash_test.c:47).</span>
<span class="go">42 if (!strcmp(how, &quot;div0&quot;)) {</span>
<span class="go">43</span>
<span class="go">44 val1 = 42;</span>
<span class="go">45 val2 = 0;</span>
<span class="go">46</span>
<span class="go">47 val3 = val1 / val2;</span>
<span class="go">48 console_printf(&quot;42/0 = %d\n&quot;, val3);</span>
<span class="go">49 } else if (!strcmp(how, &quot;jump0&quot;)) {</span>
<span class="go">50 ((void (*)(void))0)();</span>
<span class="go">51 } else if (!strcmp(how, &quot;ref0&quot;)) {</span>
</pre></div>
</div>
<p>You can see that the system crashed due to divide-by-zero.</p>
<div class="section" id="crash-with-verbose-location">
<h3>Crash with Verbose Location<a class="headerlink" href="#crash-with-verbose-location" title="Permalink to this headline"></a></h3>
<p>We often call assert(), testing specific conditions which should be met
for program execution to continue. If the condition fails, we reset with a
message listing the instruction where assert() was called from.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">829008 compat&gt; Assert @ 0x14e9f</span>
<span class="go">831203 Unhandled interrupt (2), exception sp 0x20001d20</span>
</pre></div>
</div>
<p>This address (here 0x14e9f) should then be used to find the line in
the program where it happened. Note that this address is specific to your
binary. So you have to do the search to locate the specific file/line using
your .elf file.</p>
<p>You can also enable more verbose report by setting syscfg variable
OS_CRASH_FILE_LINE to 1. The call to assert() from above would then look
like so:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">000230 compat&gt; Assert @ 0x1503b - repos/apache-mynewt-core/test/crash_test/src/crash_test.c:54</span>
<span class="go">001462 Unhandled interrupt (2), exception sp 0x20001d20</span>
</pre></div>
</div>
<p>Note that this will increase the program text size, as we now need to
store the names for the files which call assert().</p>
</div>
<div class="section" id="crash-with-console-with-stacktrace">
<h3>Crash with Console with Stacktrace<a class="headerlink" href="#crash-with-console-with-stacktrace" title="Permalink to this headline"></a></h3>
<p>Quite often you need a deeper view of the call chain to figure out
how the program got to place where it crashed. Especially if the routine
can be reached via multiple call chain routes.</p>
<p>You can enable a dump of possible call chain candidates by setting
syscfg variable OS_CRASH_STACKTRACE to 1.</p>
<p>When crash happens, the dumping routine walks backwards in the stack,
and prints any address which falls within text region. All these values
are candidates of being part of the call chain. However, you need to
filter it yourself to figure out which values are actually part of it, and
which addresses just happen to be in the stack at that time.</p>
<p>Here is a sample output:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">004067 Unhandled interrupt (3), exception sp 0x20001d28</span>
<span class="go">004067 r0:0x00000000 r1:0x00017c55 r2:0x00000000 r3:0x0000002a</span>
<span class="go">004067 r4:0x20001dc8 r5:0x00000000 r6:0x200034e4 r7:0x20003464</span>
<span class="go">004067 r8:0x2000349c r9:0x20001f08 r10:0x00017cc8 r11:0x00000000</span>
<span class="go">004067 r12:0x00000000 lr:0x00014ef9 pc:0x00014f28 psr:0x61000000</span>
<span class="go">004067 ICSR:0x00421803 HFSR:0x40000000 CFSR:0x02000000</span>
<span class="go">004067 BFAR:0xe000ed38 MMFAR:0xe000ed34</span>
<span class="go">004067 task:main</span>
<span class="go">004067 0x20001d64: 0x0001501b</span>
<span class="go">004067 0x20001d68: 0x00017338</span>
<span class="go">004067 0x20001dd0: 0x00017cc8</span>
<span class="go">004067 0x20001dd4: 0x00015c2b</span>
<span class="go">004067 0x20001dd8: 0x00015c1d</span>
<span class="go">004067 0x20001de4: 0x0001161d</span>
<span class="go">004067 0x20001df4: 0x00011841</span>
<span class="go">004067 0x20001e04: 0x000117c9</span>
<span class="go">004067 0x20001e0c: 0x00013d8f</span>
<span class="go">004067 0x20001e20: 0x00014375</span>
<span class="go">004067 0x20001e4c: 0x00013dfd</span>
<span class="go">004067 0x20001e54: 0x00013e09</span>
<span class="go">004067 0x20001e58: 0x00013e01</span>
<span class="go">004067 0x20001e5c: 0x0000925d</span>
<span class="go">004067 0x20001e64: 0x00008933</span>
<span class="go">004067 0x20001e7c: 0x00008cc3</span>
<span class="go">004067 0x20001e98: 0x000169e0</span>
<span class="go">004067 0x20001e9c: 0x00008cb9</span>
<span class="go">004067 0x20001ea0: 0x000088a9</span>
</pre></div>
</div>
<p>You can see that it contains the usual dump of registers in the beginning,
followed by addresses to stack, and the value at that location. Note that
we skip the area of memory within stack which contains the stashed register
values. This means that $pc and $lr values will not show up in the array
of addresses. Depending on your CPU architecture, different stuff gets
pushed to stack. Given where Mynewt is ported to, there’s always function
return address pushed there.</p>
<p>You would then take these values, and see if they seem legit. Here we’ll
show what to do with the output above.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>arm-elf-linux-gdb<span class="w"> </span>bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf
<span class="go">GNU gdb (GDB) 7.8.1</span>
<span class="go">Copyright (C) 2014 Free Software Foundation, Inc.</span>
<span class="go">...</span>
<span class="go">No symbol table is loaded. Use the &quot;file&quot; command.</span>
<span class="go">Reading symbols from bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf...done.</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x0001501b</span>
<span class="go">0x1501b is in crash_test_nmgr_write (repos/apache-mynewt-core/test/crash_test/src/crash_nmgr.c:68).</span>
<span class="go">63 if (rc != 0) {</span>
<span class="go">64 return MGMT_ERR_EINVAL;</span>
<span class="go">65 }</span>
<span class="go">66</span>
<span class="go">67 rc = crash_device(tmp_str); &lt;--- That is likely part of it</span>
<span class="go">68 if (rc != 0) {</span>
<span class="go">69 return MGMT_ERR_EINVAL;</span>
<span class="go">70 }</span>
<span class="go">71</span>
<span class="go">72 rc = mgmt_cbuf_setoerr(cb, 0);</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00017338 &lt;--- not relevant</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00017cc8 &lt;--- neither is this</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00015c2b</span>
<span class="go">0x15c2b is in cbor_mbuf_writer (repos/apache-mynewt-core/encoding/tinycbor/src/cbor_mbuf_writer.c:32).</span>
<span class="go">27 {</span>
<span class="go">28 int rc;</span>
<span class="go">29 struct cbor_mbuf_writer *cb = (struct cbor_mbuf_writer *) arg;</span>
<span class="go">30</span>
<span class="go">31 rc = os_mbuf_append(cb-&gt;m, data, len); &lt;-- Does not seem likely. This probably ended here due to an earlier work that this task was doing.</span>
<span class="go">32 if (rc) {</span>
<span class="go">33 return CborErrorOutOfMemory;</span>
<span class="go">34 }</span>
<span class="go">35 cb-&gt;enc.bytes_written += len;</span>
<span class="go">36 return CborNoError;</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00015c1d</span>
<span class="go">0x15c1d is in cbor_mbuf_writer (repos/apache-mynewt-core/encoding/tinycbor/src/cbor_mbuf_writer.c:27).</span>
<span class="go">22 #include &lt;tinycbor/cbor.h&gt;</span>
<span class="go">23 #include &lt;tinycbor/cbor_mbuf_writer.h&gt;</span>
<span class="go">24</span>
<span class="go">25 int</span>
<span class="go">26 cbor_mbuf_writer(struct cbor_encoder_writer *arg, const char *data, int len)</span>
<span class="go">27 {</span>
<span class="go">28 int rc; &lt;---- Nope. Not relevant.</span>
<span class="go">29 struct cbor_mbuf_writer *cb = (struct cbor_mbuf_writer *) arg;</span>
<span class="go">30</span>
<span class="go">31 rc = os_mbuf_append(cb-&gt;m, data, len);</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x0001161d</span>
<span class="go">0x1161d is in create_container (repos/apache-mynewt-core/encoding/tinycbor/src/cborencoder.c:244).</span>
<span class="go">239 memcpy(where, &amp;v, sizeof(v));</span>
<span class="go">240 }</span>
<span class="go">241</span>
<span class="go">242 static inline CborError append_to_buffer(CborEncoder *encoder, const void *data, size_t len)</span>
<span class="go">243 {</span>
<span class="go">244 return encoder-&gt;writer-&gt;write(encoder-&gt;writer, data, len); &lt;--- Hmmm, unlikely</span>
<span class="go">245 }</span>
<span class="go">246</span>
<span class="go">247 static inline CborError append_byte_to_buffer(CborEncoder *encoder, uint8_t byte)</span>
<span class="go">248 {</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00011841</span>
<span class="go">0x11841 is in preparse_value (repos/apache-mynewt-core/encoding/tinycbor/src/cborparser.c:182).</span>
<span class="go">177 /* are we at the end? */</span>
<span class="go">178 if (it-&gt;offset == parser-&gt;end)</span>
<span class="go">179 return CborErrorUnexpectedEOF;</span>
<span class="go">180</span>
<span class="go">181 uint8_t descriptor = parser-&gt;d-&gt;get8(parser-&gt;d, it-&gt;offset); &lt;--- Probably not relevant.</span>
<span class="go">182 uint8_t type = descriptor &amp; MajorTypeMask;</span>
<span class="go">183 it-&gt;type = type;</span>
<span class="go">184 it-&gt;flags = 0;</span>
<span class="go">185 it-&gt;extra = (descriptor &amp;= SmallValueMask);</span>
<span class="go">186</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x000117c9</span>
<span class="go">0x117c9 is in cbor_encoder_create_map (repos/apache-mynewt-core/encoding/tinycbor/src/cborencoder.c:521).</span>
<span class="go">516 */</span>
<span class="go">517 CborError cbor_encoder_create_map(CborEncoder *encoder, CborEncoder *mapEncoder, size_t length)</span>
<span class="go">518 {</span>
<span class="go">519 if (length != CborIndefiniteLength &amp;&amp; length &gt; SIZE_MAX / 2)</span>
<span class="go">520 return CborErrorDataTooLarge;</span>
<span class="go">521 return create_container(encoder, mapEncoder, length, MapType &lt;&lt; MajorTypeShift); &lt;-- I don&#39;t think this is relevant either.</span>
<span class="go">522 }</span>
<span class="go">523</span>
<span class="go">524 /**</span>
<span class="go">525 * Creates a indefinite-length byte string in the CBOR stream provided by</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">list *0x00013d8f</span>
<span class="go">0x13d8f is in nmgr_handle_req (repos/apache-mynewt-core/mgmt/newtmgr/src/newtmgr.c:261).</span>
<span class="go">256 } else {</span>
<span class="go">257 rc = MGMT_ERR_ENOENT;</span>
<span class="go">258 }</span>
<span class="go">259 } else if (hdr.nh_op == NMGR_OP_WRITE) {</span>
<span class="go">260 if (handler-&gt;mh_write) {</span>
<span class="go">261 rc = handler-&gt;mh_write(&amp;nmgr_task_cbuf.n_b); &lt;-- This is part of it.</span>
<span class="go">262 } else {</span>
<span class="go">263 rc = MGMT_ERR_ENOENT;</span>
<span class="go">264 }</span>
<span class="go">265 } else {</span>
</pre></div>
</div>
<p>I was sending a newtmgr command which crashed the system. As you can see from this this example, little less than 50% of the addresses were part of the call chain. So read this output with care.</p>
</div>
</div>
<div class="section" id="coredump">
<h2>Coredump<a class="headerlink" href="#coredump" title="Permalink to this headline"></a></h2>
<p>Coredump contains a full dump of system memory, and CPU registers. You can inspect the system state, including stack of the failing task, or any of the global state.</p>
<p>You can enable coredumps by setting syscfg variable OS_COREDUMP to 1. When crash happens, dumper will write the contents to flash_map region COREDUMP_FLASH_AREA.
After the crash, you can use imgmgr to download the coredump for offline analysis. To enable download/erase commands, you need to set syscfg variable IMGMGR_COREDUMP to 1.</p>
<p>Coredump package does not overwrite a previous coredump, if it exists in the flash. To get a new one, you first need to erase the area.</p>
<p>You can either use a dedicated area of flash, or use image slot 1 as the target.
If image slot1 and coredump areas are co-located, coredump will overwrite the image, unless image upgrade is in progress (slot1 marked as pending, or slot0 is not confirmed).</p>
<div class="section" id="fetching-coredump-with-newtmgr">
<h3>Fetching Coredump with newtmgr<a class="headerlink" href="#fetching-coredump-with-newtmgr" title="Permalink to this headline"></a></h3>
<p>newtmgr coredump management commands are handled by imgmgr. You can query whether corefile is present, download one if it exists, and then erase the area afterwards.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>newtmgr<span class="w"> </span>-c<span class="w"> </span>ttys005<span class="w"> </span>image<span class="w"> </span>corelist
<span class="go">Corefile present</span>
<span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>newtmgr<span class="w"> </span>-c<span class="w"> </span>ttys005<span class="w"> </span>image<span class="w"> </span>coredownload<span class="w"> </span>-e<span class="w"> </span>core.elf
<span class="go">0</span>
<span class="go">512</span>
<span class="go">1024</span>
<span class="go">1536</span>
<span class="go"> ...</span>
<span class="go">65676</span>
<span class="go">Done writing core file to core.elf; hash=7e20dcdd136c6796fcb2a51e7384e90800d2ec045f2ee088af32529e929e2130</span>
<span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>newtmgr<span class="w"> </span>-c<span class="w"> </span>ttys005<span class="w"> </span>image<span class="w"> </span>coreerase
<span class="go">Done</span>
</pre></div>
</div>
<p>I specified option ‘-e’ to coredownload command. This converts the internal data representation to ELF file; a format that gdb understands.</p>
</div>
<div class="section" id="coredump-analysis-offline">
<h3>Coredump Analysis Offline<a class="headerlink" href="#coredump-analysis-offline" title="Permalink to this headline"></a></h3>
<p>Not all GDB configurations support corefiles. Specifically for the
architecture we’re showing here, ‘arm-none-eabi-gdb’ does not have it
built in. However, ‘arm-elf-linux-gdb’ does.</p>
<p>Here’s how I built it with MacOS:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">tar xvzf gdb-7.8.1.tar.gz</span>
<span class="go">cd gdb-7.8.1</span>
<span class="go">./configure --target=arm-elf-linux --without-lzma --without-guile --without-libunwind-ia64 --with-zlib</span>
<span class="go">make</span>
<span class="go">gdb/gdb -v</span>
</pre></div>
</div>
<p>And this is how I built it for Linux:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">sudo apt-get install zlibc zlib1g zlib1g-dev libexpat-dev libncurses5-dev liblzma-dev</span>
<span class="go">tar xvzf gdb-7.8.1.tar.gz</span>
<span class="go">cd gdb-7.8.1</span>
<span class="go">./configure --target=arm-elf-linux --with-lzma --with-expat --without-libunwind-ia64 --with-zlib --without-babeltrace</span>
<span class="go">make</span>
<span class="go">gdb/gdb -v</span>
</pre></div>
</div>
<p>Now that I have a suitable version of gdb at hand, I can use it to analyze
the corefile.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>arm-elf-linux-gdb<span class="w"> </span>bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf<span class="w"> </span>core.elf
<span class="go">GNU gdb (GDB) 7.8.1</span>
<span class="go">Copyright (C) 2014 Free Software Foundation, Inc.</span>
<span class="go">License GPLv3+: GNU GPL version 3 or later &lt;http://gnu.org/licenses/gpl.html&gt;</span>
<span class="go">This is free software: you are free to change and redistribute it.</span>
<span class="go">There is NO WARRANTY, to the extent permitted by law. Type &quot;show copying&quot;</span>
<span class="go">and &quot;show warranty&quot; for details.</span>
<span class="go">This GDB was configured as &quot;--host=x86_64-apple-darwin14.5.0 --target=arm-elf-linux&quot;.</span>
<span class="go">Type &quot;show configuration&quot; for configuration details.</span>
<span class="go">For bug reporting instructions, please see:</span>
<span class="go">&lt;http://www.gnu.org/software/gdb/bugs/&gt;.</span>
<span class="go">Find the GDB manual and other documentation resources online at:</span>
<span class="go">&lt;http://www.gnu.org/software/gdb/documentation/&gt;.</span>
<span class="go">For help, type &quot;help&quot;.</span>
<span class="go">Type &quot;apropos word&quot; to search for commands related to &quot;word&quot;...</span>
<span class="go">No symbol table is loaded. Use the &quot;file&quot; command.</span>
<span class="go">Reading symbols from bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf...done.</span>
<span class="go">[New process 1]</span>
<span class="gp">#</span><span class="m">0</span><span class="w"> </span>0x00014f28<span class="w"> </span><span class="k">in</span><span class="w"> </span>crash_device<span class="w"> </span><span class="o">(</span>
<span class="go"> how=how@entry=0x20001dc8 &lt;os_main_stack+3904&gt; &quot;div0&quot;)</span>
<span class="go"> at repos/apache-mynewt-core/test/crash_test/src/crash_test.c:47</span>
<span class="go">47 val3 = val1 / val2;</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">bt</span>
<span class="gp">#</span><span class="m">0</span><span class="w"> </span>0x00014f28<span class="w"> </span><span class="k">in</span><span class="w"> </span>crash_device<span class="w"> </span><span class="o">(</span>
<span class="go"> how=how@entry=0x20001dc8 &lt;os_main_stack+3904&gt; &quot;div0&quot;)</span>
<span class="go"> at repos/apache-mynewt-core/test/crash_test/src/crash_test.c:47</span>
<span class="gp">#</span><span class="m">1</span><span class="w"> </span>0x0001501a<span class="w"> </span><span class="k">in</span><span class="w"> </span>crash_test_nmgr_write<span class="w"> </span><span class="o">(</span><span class="nv">cb</span><span class="o">=</span>0x20003464<span class="w"> </span>&lt;nmgr_task_cbuf&gt;<span class="o">)</span>
<span class="go"> at repos/apache-mynewt-core/test/crash_test/src/crash_nmgr.c:67</span>
<span class="gp">#</span><span class="m">2</span><span class="w"> </span>0x00013d8e<span class="w"> </span><span class="k">in</span><span class="w"> </span>nmgr_handle_req<span class="w"> </span><span class="o">(</span>
<span class="go"> nt=nt@entry=0x200034e4 &lt;nmgr_shell_transport&gt;,</span>
<span class="go"> req=0x20002014 &lt;os_msys_init_1_data+292&gt;)</span>
<span class="go"> at repos/apache-mynewt-core/mgmt/newtmgr/src/newtmgr.c:261</span>
<span class="gp">#</span><span class="m">3</span><span class="w"> </span>0x00013dfc<span class="w"> </span><span class="k">in</span><span class="w"> </span>nmgr_process<span class="w"> </span><span class="o">(</span><span class="nv">nt</span><span class="o">=</span>0x200034e4<span class="w"> </span>&lt;nmgr_shell_transport&gt;<span class="o">)</span>
<span class="go"> at repos/apache-mynewt-core/mgmt/newtmgr/src/newtmgr.c:325</span>
<span class="gp">#</span><span class="m">4</span><span class="w"> </span>0x00013e08<span class="w"> </span><span class="k">in</span><span class="w"> </span>nmgr_event_data_in<span class="w"> </span><span class="o">(</span><span class="nv">ev</span><span class="o">=</span>&lt;optimized<span class="w"> </span>out&gt;<span class="o">)</span>
<span class="go"> at repos/apache-mynewt-core/mgmt/newtmgr/src/newtmgr.c:332</span>
<span class="gp">#</span><span class="m">5</span><span class="w"> </span>0x0000925c<span class="w"> </span><span class="k">in</span><span class="w"> </span>os_eventq_run<span class="w"> </span><span class="o">(</span><span class="nv">evq</span><span class="o">=</span>&lt;optimized<span class="w"> </span>out&gt;<span class="o">)</span>
<span class="go"> at repos/apache-mynewt-core/kernel/os/src/os_eventq.c:162</span>
<span class="gp">#</span><span class="m">6</span><span class="w"> </span>0x00008932<span class="w"> </span><span class="k">in</span><span class="w"> </span>main<span class="w"> </span><span class="o">(</span><span class="nv">argc</span><span class="o">=</span>&lt;optimized<span class="w"> </span>out&gt;,<span class="w"> </span><span class="nv">argv</span><span class="o">=</span>&lt;optimized<span class="w"> </span>out&gt;<span class="o">)</span>
<span class="go"> at repos/apache-mynewt-core/apps/slinky/src/main.c:289</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">p g_current_task</span>
<span class="gp">$</span><span class="nv">1</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">(</span>struct<span class="w"> </span>os_task<span class="w"> </span>*<span class="o">)</span><span class="w"> </span>0x20001e88<span class="w"> </span>&lt;os_main_task&gt;
</pre></div>
</div>
<p>Setting up coredumps, setting up a mechanism to download them, managing
them on the device and building gdb is certainly more work, but you’ll
have much more data available when you start analyzing problems.
I don’t think I can emphasize too much how valuable having all this data
is, when you’re stuck with a difficult-to-solve crash.</p>
</div>
</div>
<div class="section" id="task-stack-use">
<h2>Task Stack Use<a class="headerlink" href="#task-stack-use" title="Permalink to this headline"></a></h2>
<p>One of the common problems is stack overflow within a task. Mynewt OS fills
the stack of a task with a pattern at the time task gets created. If you’re
suspecting that stack might be blown, check the start of the stack (and
memory right before it).</p>
<p>There are also APIs to check the stack use. Task statistics, accessible
with console CLI or newtmgr, attempt to report the stack use by the tasks.
They do it by inspecting the presence of this pattern, so it is possible
to fool them also.</p>
<p>Here is what that data looks like at console. ‘stksz’ is the amount of
stack, and ‘stkuse’ is how much of it we think the task used it.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">7109237 compat&gt; tasks</span>
<span class="go">7109387 Tasks:</span>
<span class="go">7109387 task pri tid runtime csw stksz stkuse lcheck ncheck flg</span>
<span class="go">7109390 idle 255 0 7109375 72810 64 26 0 0</span>
<span class="go">7109392 main 127 1 15 6645 1024 388 0 0</span>
<span class="go">7109394 task1 8 2 0 55543 192 116 0 0</span>
<span class="go">7109396 task2 9 3 0 55543 64 28 0 0</span>
</pre></div>
</div>
<p>There are some helper macros for this when using gdb also. You’ll need
to eyeball the stack manually, but at least it will tell you where to
find the stacks.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">[marko@IsMyLaptop:~/src2/incubator-mynewt-blinky]$ </span>newt<span class="w"> </span>debug<span class="w"> </span>slinky_nrf52
<span class="go">Debugging bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf</span>
<span class="go"> ...</span>
<span class="go">Reading symbols from bin/targets/slinky_nrf52/app/apps/slinky/slinky.elf...done.</span>
<span class="go">os_tick_idle (ticks=128)</span>
<span class="go"> at repos/apache-mynewt-core/hw/mcu/nordic/nrf52xxx/src/hal_os_tick.c:164</span>
<span class="go">164 if (ticks &gt; 0) {</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">os_tasks</span>
<span class="go">Undefined command: &quot;os_tasks&quot;. Try &quot;help&quot;.</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">source repos/apache-mynewt-core/compiler/gdbmacros/os.gdb</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">os_tasks</span>
<span class="go"> prio state stack stksz task name</span>
<span class="go">* 255 0x1 0x20000e88 64 0x200035fc idle</span>
<span class="go"> 127 0x2 0x20001e88 1024 0x20001e88 main</span>
<span class="go"> 8 0x2 0x20003ab0 192 0x20000cbc task1</span>
<span class="go"> 9 0x2 0x20003bc0 64 0x20000d0c task2</span>
</pre></div>
</div>
<div class="section" id="stack-check-during-context-switch">
<h3>Stack Check during Context Switch<a class="headerlink" href="#stack-check-during-context-switch" title="Permalink to this headline"></a></h3>
<p>You can ask OS to check the top of the stack on every context switch.
It’ll walk over X number of os_stack_t elements, checking if our pattern
is still there for the task which is about to be switched out.</p>
<p>You can enable this by setting syscfg variable OS_CTX_SW_STACK_CHECK to 1.
Another syscfg variable, OS_CTX_SW_STACK_GUARD, controls how far into
the stack we peek.</p>
</div>
</div>
<div class="section" id="memory-corruption">
<h2>Memory Corruption<a class="headerlink" href="#memory-corruption" title="Permalink to this headline"></a></h2>
<p>The recommended way of allocating blocks of memory is by using the
OS memory pools. As with all dynamically allocated memory, there can
issues with using after it gets freed, or it does not get freed at
all, code tries to free the same element multiple times, or you end up
overwriting the data within the next element.</p>
<p>There are syscfg variables you can set to get OS to do some of
sanity checks for you.</p>
<div class="section" id="sanity-check-on-free">
<h3>Sanity Check on Free<a class="headerlink" href="#sanity-check-on-free" title="Permalink to this headline"></a></h3>
<p>By setting syscfg variable OS_MEMPOOL_CHECK to 1, OS will act when
an element is freed. It will walk through the list of already freed
entries, and check that the entry being freed does not already appear
in the free list. It will also check that the pool entry the code is
trying to free is within the memory range belonging to that pool.</p>
</div>
<div class="section" id="poisoning-pool">
<h3>Poisoning Pool<a class="headerlink" href="#poisoning-pool" title="Permalink to this headline"></a></h3>
<p>If you set syscfg variable OS_MEMPOOL_POISON, the OS fills pool
entry memory with a data pattern whenever that entry is being freed.
It then checks that this pattern has not been disturbed when that
pool entry is allocated again. This would help to detect the scenarios
when code is still using a freed pool entry. Of course, the detection
will only be triggered if the code tries to modify the data belonging
tot that entry.</p>
</div>
<div class="section" id="pool-guard-areas">
<h3>Pool Guard Areas<a class="headerlink" href="#pool-guard-areas" title="Permalink to this headline"></a></h3>
<p>Normally OS takes the memory region allocated for pool use and
splits it into chunks which are exactly the size of the pool entry (taking
into account the alignment restriction). By setting syscfg variable
OS_MEMPOOL_GUARD to 1, it creates a ‘guard area’ between pool elements,
and writes a specific data pattern to it. This is then checked whenever
the pool entry is freed, or allocated. If code writes past the data area
belonging to pool entry, the pattern would likely be disturbed, and would
cause an assert.</p>
</div>
<div class="section" id="exhausting-memory">
<h3>Exhausting Memory<a class="headerlink" href="#exhausting-memory" title="Permalink to this headline"></a></h3>
<p>Memory pool usage can be monitored through the mempool API. The data
collected there can be seen via the console CLI and/or using newtmgr.</p>
<p>Here is that data as seen from console:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">7375292 compat&gt; mpool</span>
<span class="go">7375581 Mempools:</span>
<span class="go">7375581 name blksz cnt free min</span>
<span class="go">7375584 msys_1 292 12 12 8</span>
<span class="go">7375585 modlog_mapping_pool 12 16 12 12</span>
</pre></div>
</div>
<p>We list the memory pools OS knows about. ‘cnt’ tells how many
elements the pool has in total. ‘free’ tells how many elements are
currently in the free pool, and ‘min’ tells what’s the low point
for the number of elements within the pool.</p>
<p>What you should check is the ‘min’ number. If it gets to 0, it means
that the pool has been exhausted at one point. Depending on what
the pool is used for, you might want to take action.</p>
<p>There’s some gdb helper macros to inspect the data belonging to
the default networking memory pool, msys_1.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">source repos/apache-mynewt-core/compiler/gdbmacros/mbuf.gdb</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">mn_msys1_print</span>
<span class="go">Mbuf addr: 0x20001ef0</span>
<span class="go">Mbuf header: $1 = {om_data = 0x20002380 &lt;os_msys_init_1_data+1168&gt; &quot;\244$&quot;,</span>
<span class="go"> om_flags = 0 &#39;\000&#39;, om_pkthdr_len = 8 &#39;\b&#39;, om_len = 14,</span>
<span class="go"> om_omp = 0x20002ca0 &lt;os_msys_init_1_mbuf_pool&gt;, om_next = {sle_next = 0x0},</span>
<span class="go"> om_databuf = 0x20001f00 &lt;os_msys_init_1_data+16&gt; &quot;\016&quot;}</span>
<span class="go">Packet header: $2 = {omp_len = 14, omp_flags = 0, omp_next = {stqe_next = 0x0}}</span>
<span class="go"> ....</span>
<span class="go">---Type &lt;return&gt; to continue, or q &lt;return&gt; to quit---q</span>
<span class="go">Quit</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">help mn_msys1_print</span>
<span class="go">usage: mn_msys1_print</span>
<span class="go">Prints all mbufs in the first msys pool. Both allocated and unallocated mbufs</span>
<span class="go">are printed.</span>
<span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">mn_msys1_free_print</span>
<span class="go">Mbuf addr: 0x20002138</span>
<span class="go"> ....</span>
<span class="go">Mbuf addr: 0x20002b7c</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="crash-in-log">
<h2>Crash in Log<a class="headerlink" href="#crash-in-log" title="Permalink to this headline"></a></h2>
<p>System restarts can also be recorded in reboot log. Assuming reboot log
has been set up, it’ll write a record either when system is asked
to reset (managed reset), or when system is coming back after an
unexpected reset.</p>
<p>To facilitate the latter, you should include a call from main()
to reboot package, reporting what HAL is telling you.</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="kt">int</span>
<span class="nf">main</span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">argc</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">**</span><span class="n">argv</span><span class="p">)</span>
<span class="p">{</span>
<span class="w"> </span><span class="n">sysinit</span><span class="p">();</span>
<span class="w"> </span><span class="cm">/* .... */</span>
<span class="w"> </span><span class="n">reboot_start</span><span class="p">(</span><span class="n">hal_reset_cause</span><span class="p">());</span>
</pre></div>
</div>
<p>This will then show if the system was restarted due to POR, brownout,
or hardware watchdog (the cause determination is MCU specific, so YMMV).
As you remember, exit via assert() and unhandled interrupt is going
via the dumper. So these kind of reboots will show up as ‘SOFT’ reset.</p>
<p>Restarts due to hardware watchdog will show up here, however. You can also
add a printout to beginning of main(), outputting what hal_reset_cause_str()
returns. That will generate a report on the console.</p>
</div>
</div>
</div>
</div>
<div class="rst-footer-buttons row" role="navigation" aria-label="footer navigation">
<a href="../other/other.html" class="btn btn-neutral float-right" title="Other" accesskey="n">Next: Other <span class="fa fa-arrow-circle-right"></span></a>
<a href="segger_sysview.html" class="btn btn-neutral" title="SEGGER SystemView" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous: SEGGER SystemView</a>
</div>
</div>
</div>
</div>
<!-- ENDS CONTENT SECTION -->
</div>
<!-- ENDS .content -->
</div>
</div>
<footer>
<div class="container">
<div class="row">
<div class="col-xs-12">
<p class="copyright">Apache Mynewt is available under Apache License, version 2.0.</p>
</div>
<div class="col-xs-12">
<div class="logos">
<img src="../../_static/img/asf_logo_wide_small.png" alt="Apache" title="Apache">
<small class="footnote">
Apache Mynewt, Mynewt, Apache, the Apache feather logo, and the Apache Mynewt project logo are either
registered trademarks or trademarks of the Apache Software Foundation in the United States and other countries.
</small>
<a href="">
<img src="../../_static/img/add_to_slack.png" alt="Slack Icon" title="Join our Slack Community" />
</a>
</div>
</div>
</div>
</div>
</footer>
</div>
<!-- ENDS #wrapper -->
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'latest',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt',
LINK_SUFFIX: '.html'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/bootstrap-3.0.3.min.js"></script>
<script type="text/javascript" src="../../_static/js/affix.js"></script>
<script type="text/javascript" src="../../_static/js/main.js"></script>
</body>
</html>