海思hi3536平台内核报错 一直打印错误提示 SYS_GetLocalCurPts hi_sched_clock err...

本帖最后由 sanmaoljh 于 2018-10-17 15:36 编辑

环境                ：hi3536  +  Hi3536_SDK_V2.0.6.0.tgz

问题描述：
有部分机器，不定时随机出现这个内核报错，上层应用qt就会卡死，运行一段时间1天或者几天会出现，请问下应该 怎么去分析，谢谢！

海思hi3536平台内核报错 一直打印错误提示
SYS_GetLocalCurPts hi_sched_clock error u64TimeNow=244900145529808,u64TimeLast=244934488595280

同时通过top命令查看，发现cpu0和cpu3的使用率100%，软中断sirq也是100%，请问怎么回事？谢谢！







因为这个是概率性出现，又很难模拟重现这个问题
根据内核错误栈打印信息，怀疑指向网卡stmmac有关，又因为stmmac.ko是海思厂家提供，没法看到源码，现在不知道怎么入手解决，


查看内核奔溃日志，都是指向stmmac模块，是不是与网卡有关，谢谢！
因为内核提供的日志信息太少，所以没办法准确定位。谢谢！
同时网上搜索资料，发现有网友出现这个问题，在hi3516，hi3531,等其他硬件平台都出现过，我想应该是一个共性的问题，但是都没有下文解决
[url]http://bbs.ebaina.com/forum.php?mod=viewthread&tid=18138&highlight=hi%5C_sched%5C_clock%2Berro[/url]

现在猜测，是不是网络数据量大造成网卡偶尔卡死在里面死循环？


查看
~ # dmesg
9d30  00000001 00000020 00000001 0007a120 00000000 00000028 00989680 00004e20
9d50  00000000 00000000 00000000 00000000 00000000 00000001 00000001 00000004
9d70  00000001 00000000 00000000 00000000 00000001 00000001 00000001 00000001
9d90  00000001 00000000 00000000 00000000 000186a0 00000000 00000001 000003e8
9db0  00000204 00000000 0003eb28 00001942 00001942 01ffffff 00000001 00000001

R6: 0xc05dedec:
edec  00000010 00000020 00000040 00000080 00000100 00000200 00000400 00000800
ee0c  00001000 00002000 00004000 00008000 00010000 00020000 00040000 00080000
ee2c  00100000 00200000 00400000 00800000 01000000 02000000 04000000 08000000
ee4c  10000000 20000000 40000000 80000000 7570635f 0070755f c0809cfc c0809cf4
ee6c  c0809cf0 c0809cf8 0000000f c002d474 c002cf48 c002d440 c002d0a4 00000000
ee8c  c00de508 c00de618 00000000 00000000 00000000 00000000 00000000 00000000
eeac  00000000 00000000 c002d074 00000000 c00dea80 00000000 00000000 00000000
eecc  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000

R7: 0xd9077f80:
7f80  60000013 ffffffff d9077fcc c083a2a5 d9076000 c0012bc0 c108c930 00000000
7fa0  6b080e86 00000000 c083a2a5 c0809d50 c05dee6c d9076000 00000000 01400000
7fc0  c07f7930 c007e4e0 c083a2a5 c0809d50 c05dee6c d9076000 c083a2a5 d9076000
7fe0  6b080e88 c005bf18 00000000 c083a758 00000000 405d03e4 9ef49fe8 e67bf4ff
8000  00000000 00000001 00000000 d906d000 c080fcc4 00000002 00000015 c1095ac0
8020  d906d000 c07f8ac0 d9078038 d9283340 d92a0000 00000000 d9079fdc d9079f38
8040  c05db404 00000000 00000000 00000000 00000000 00000000 00000000 00000000
8060  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000

R8: 0xc083a225:
a224  00000000 c05d30dc c083a240 00000000 c05d3240 c083a1b0 00000064 c05d32fc
a244  c087afdc 00000000 c04a8dc0 c083a0c0 00000000 00000000 00000000 00000000
a264  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a284  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a2a4  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a2c4  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a2e4  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a304  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a324  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000

R9: 0xd9077f80:
7f80  60000013 ffffffff d9077fcc c083a2a5 d9076000 c0012bc0 c108c930 00000000
7fa0  6b080e86 00000000 c083a2a5 c0809d50 c05dee6c d9076000 00000000 01400000
7fc0  c07f7930 c007e4e0 c083a2a5 c0809d50 c05dee6c d9076000 c083a2a5 d9076000
7fe0  6b080e88 c005bf18 00000000 c083a758 00000000 405d03e4 9ef49fe8 e67bf4ff
8000  00000000 00000001 00000000 d906d000 c080fcc4 00000002 00000015 c1095ac0
8020  d906d000 c07f8ac0 d9078038 d9283340 d92a0000 00000000 d9079fdc d9079f38
8040  c05db404 00000000 00000000 00000000 00000000 00000000 00000000 00000000
8060  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000

R10: 0xd9077f80:
7f80  60000013 ffffffff d9077fcc c083a2a5 d9076000 c0012bc0 c108c930 00000000
7fa0  6b080e86 00000000 c083a2a5 c0809d50 c05dee6c d9076000 00000000 01400000
7fc0  c07f7930 c007e4e0 c083a2a5 c0809d50 c05dee6c d9076000 c083a2a5 d9076000
7fe0  6b080e88 c005bf18 00000000 c083a758 00000000 405d03e4 9ef49fe8 e67bf4ff
8000  00000000 00000001 00000000 d906d000 c080fcc4 00000002 00000015 c1095ac0
8020  d906d000 c07f8ac0 d9078038 d9283340 d92a0000 00000000 d9079fdc d9079f38
8040  c05db404 00000000 00000000 00000000 00000000 00000000 00000000 00000000
8060  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
CPU: 2 PID: 0 Comm: swapper/2 Tainted: P           O 3.10.0_hi3536 #114
[] (unwind_backtrace+0x0/0xf4) from [] (show_stack+0x10/0x14)
[] (show_stack+0x10/0x14) from [] (handle_IPI+0xf8/0x1b0)
[] (handle_IPI+0xf8/0x1b0) from [] (gic_handle_irq+0x88/0x8c)
[] (gic_handle_irq+0x88/0x8c) from [] (__irq_svc+0x40/0x50)
Exception stack(0xd9079f98 to 0xd9079fe0)
9f80:                                                       c1094930 00000000
9fa0: 0621ac80 00000000 c083a2a5 c0809d50 c05dee6c d9078000 c083a2a5 d9078000
9fc0: d9078000 c0809cf0 00000000 d9079fe0 c00142a4 c00142a8 60000013 ffffffff
[] (__irq_svc+0x40/0x50) from [] (arch_cpu_idle+0x2c/0x30)
[] (arch_cpu_idle+0x2c/0x30) from [] (cpu_startup_entry+0xfc/0x140)
[] (cpu_startup_entry+0xfc/0x140) from [<405d03e4>] (0x405d03e4)
{ 3}  (t=1860309 jiffies g=301656 c=301655 q=203489)
IPI backtrace for cpu 3

CPU: 3 PID: 0 Comm: swapper/3 Tainted: P           O 3.10.0_hi3536 #114
task: d906d400 ti: d907a000 task.ti: d907a000
PC is at _raw_spin_lock+0x28/0x3c
LR is at tnk_check_alive_timer+0x20/0xcc [stmmac]
pc : []    lr : []    psr: 80000113
sp : d907be28  ip : bf65f534  fp : d9016000
r10: d9016c14  r9 : d9016a14  r8 : bf0070dc
r7 : d907a038  r6 : 00000100  r5 : d7f03c00  r4 : d6c99f80
r3 : 00003f19  r2 : 00003f1a  r1 : 00000000  r0 : d6c99fb8
Flags: Nzcv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
Control: 10c5387d  Table: 56c7c06a  DAC: 00000015

PC: 0xc05dc398:
c398  e58ce000 e88d000c eb000043 e28dd014 e8bd80f0 e3a01000 eaffffca 00000000
c3b8  e1901f9f e3a03000 e0512861 02811801 01803f91 e3530000 1afffff8 e3520000
c3d8  1a000002 f57ff05f e3a00001 e12fff1e e3a00000 e12fff1e e1903f9f e2832801
c3f8  e1801f92 e3310000 1afffffa e1a02823 e6ff3073 e1530002 0a000003 e320f002
c418  e1d030b0 e1530002 1afffffb f57ff05f e12fff1e e1a02000 e10f0000 f10c0080
c438  e1923f9f e2831801 e182cf91 e33c0000 1afffffa e1a01823 e6ff3073 e1530001
c458  0a000003 e320f002 e1d230b0 e1530001 1afffffb f57ff05f e12fff1e f10c0080
c478  e1903f9f e2832801 e1801f92 e3310000 1afffffa e1a02823 e6ff3073 e1530002

SP: 0xd907bda8:
bda8  fe40200c c080a108 d907bde0 c0854f80 fe402000 c0008568 bf0070fc c05dc418
bdc8  80000113 ffffffff d907be14 bf0070dc d9016a14 c0012bc0 d6c99fb8 00000000
bde8  00003f1a 00003f19 d6c99f80 d7f03c00 00000100 d907a038 bf0070dc d9016a14
be08  d9016c14 d9016000 bf65f534 d907be28 bf0070fc c05dc418 80000113 ffffffff
be28  d907be5c c004f588 d9016000 c0032b8c ffffffff c109c628 c109c638 c109c628
be48  d8589f40 c109c638 c109c628 3b9aca00 3b9aca00 c0047094 ffffffff 00000000
be68  c109dac0 d907bea8 0003bcab 00000000 16343b40 00000000 c0809fb0 2a031e80
be88  0000debe 00989680 00000000 c109c1c0 c061be84 00000067 0000004c 00000033

FP: 0xd9015f80:
5f80  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
5fa0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
5fc0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
5fe0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
6000  152a152a d7f03c3c 01753e21 01753e20 0000000d d9016014 d9016014 d901601c
6020  d901601c d9016024 d9016024 d901602c d901602c d9016034 d9016034 d901603c
6040  d901603c d9016044 d9016044 d901604c d901604c d9016054 d9016054 d901605c
6060  d901605c d9016064 d9016064 d901606c d901606c d9016074 d9016074 d901607c

R0: 0xd6c99f38:
9f38  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
9f58  00000000 c063543c 00000000 00000000 00000000 00000000 d6c98fc0 c0013128
9f78  d6c98000 00000000 63792a3b ed703fb7 dd267bad 682aba10 01010002 00000000
9f98  d7fe8018 d6c9b498 c083461c 0000175b d924dd68 ffffffff 00000002 00000000
9fb8  3f1b3f19 00000000 00000000 d6c99fc4 d6c99fc4 d6c99fcc d6c99fcc 00000000
9fd8  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
9ff8  00015554 00000000 d6403040 00000000 00000000 00000308 d939e180 00000000
a018  00010001 00000001 00000000 00029cc0 d6c9a028 d6c9a028 00000000 00010001

R4: 0xd6c99f00:
9f00  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
9f20  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
9f40  00000000 00000000 00000000 00000000 00000000 00000000 00000000 c063543c
9f60  00000000 00000000 00000000 00000000 d6c98fc0 c0013128 d6c98000 00000000
9f80  63792a3b ed703fb7 dd267bad 682aba10 01010002 00000000 d7fe8018 d6c9b498
9fa0  c083461c 0000175b d924dd68 ffffffff 00000002 00000000 3f1b3f19 00000000
9fc0  00000000 d6c99fc4 d6c99fc4 d6c99fcc d6c99fcc 00000000 00000000 00000000
9fe0  00000000 00000000 00000000 00000000 00000000 00000000 00015554 00000000

R5: 0xd7f03b80:
3b80  00000000 00000000 01913551 000000dc 00000000 000002d4 00000000 000002d4
3ba0  00000000 00000000 bf018fbc 000e000e 00070007 00000000 000000d0 00000000
3bc0  00000000 d7f03bc4 d7f03bc4 00000000 fffff916 0000ffff 00000001 00000000
3be0  00000000 00000000 00200200 01913b96 d9012000 bf007028 d7f03b10 ffffffff
3c00  d7f198c0 00000000 d6c99f80 d7f03c0c d7f03c0c 00000000 00cc00cc d7f03c1c
3c20  d7f03c1c 00000000 06420642 d7f03c2c d7f03c2c 00000000 00000000 00000000
3c40  00200200 01753e20 d9016000 bf0070dc d7f03c00 ffffffff 00000040 00000000
3c60  79ba9ba2 5de87358 00000000 00000000 00000000 00000000 01693bdf 000000d2

R7: 0xd9079fb8:
9fb8  00000000 01400000 c07f7930 c007e4e0 c083a2a5 c0809d50 c05dee6c d9078000
9fd8  c083a2a5 d9078000 0621aede c005bf18 00000000 c083a758 00000000 405d03e4
9ff8  7aba6b6d 5fe2d15d 00000002 00000101 00000000 d906d400 c080fcc4 00000003
a018  00000015 c109dac0 d906d400 c07f8ac0 d907a038 d9283340 d7807c00 d9283340
a038  d907bfdc d907bf38 c05db404 00000000 00000000 00000000 00000000 00000000
a058  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a078  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
a098  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000

R9: 0xd9016994:
6994  d9016994 d9016994 d901699c d901699c d90169a4 d90169a4 d90169ac d90169ac
69b4  d90169b4 d90169b4 d90169bc d90169bc d90169c4 d90169c4 d90169cc d90169cc
69d4  d90169d4 d90169d4 d90169dc d90169dc d90169e4 d90169e4 d90169ec d90169ec
69f4  d90169f4 d90169f4 d90169fc d90169fc d9016a04 d9016a04 d8aa81e8 d8aa81e8
6a14  d9016a14 d9016a14 d9016a1c d9016a1c d9016a24 d9016a24 d9016a2c d9016a2c
6a34  d9016a34 d9016a34 d9016a3c d9016a3c d9016a44 d9016a44 d9016a4c d9016a4c
6a54  d9016a54 d9016a54 d9016a5c d9016a5c d9016a64 d9016a64 d9016a6c d9016a6c
6a74  d9016a74 d9016a74 d9016a7c d9016a7c d9016a84 d9016a84 d9016a8c d9016a8c

R10: 0xd9016b94:
6b94  d9016b94 d9016b94 d9016b9c d9016b9c d9016ba4 d9016ba4 d9016bac d9016bac
6bb4  d9016bb4 d9016bb4 d9016bbc d9016bbc d9016bc4 d9016bc4 d9016bcc d9016bcc
6bd4  d9016bd4 d9016bd4 d9016bdc d9016bdc d9016be4 d9016be4 d9016bec d9016bec
6bf4  d9016bf4 d9016bf4 d9016bfc d9016bfc d9016c04 d9016c04 d9016c0c d9016c0c
6c14  d9016c14 d9016c14 d9016c1c d9016c1c d9016c24 d9016c24 d9016c2c d9016c2c
6c34  d9016c34 d9016c34 d9016c3c d9016c3c d9016c44 d9016c44 d9016c4c d9016c4c
6c54  d9016c54 d9016c54 d9016c5c d9016c5c d9016c64 d9016c64 d9016c6c d9016c6c
6c74  d9016c74 d9016c74 d9016c7c d9016c7c d9016c84 d9016c84 d9016c8c d9016c8c
CPU: 3 PID: 0 Comm: swapper/3 Tainted: P           O 3.10.0_hi3536 #114
[] (unwind_backtrace+0x0/0xf4) from [] (show_stack+0x10/0x14)
[] (show_stack+0x10/0x14) from [] (handle_IPI+0xf8/0x1b0)
[] (handle_IPI+0xf8/0x1b0) from [] (gic_handle_irq+0x88/0x8c)
[] (gic_handle_irq+0x88/0x8c) from [] (__irq_svc+0x40/0x50)
Exception stack(0xd907bde0 to 0xd907be28)
bde0: d6c99fb8 00000000 00003f1a 00003f19 d6c99f80 d7f03c00 00000100 d907a038
be00: bf0070dc d9016a14 d9016c14 d9016000 bf65f534 d907be28 bf0070fc c05dc418
be20: 80000113 ffffffff
[] (__irq_svc+0x40/0x50) from [] (_raw_spin_lock+0x28/0x3c)
[] (_raw_spin_lock+0x28/0x3c) from [] (tnk_check_alive_timer+0x20/0xcc [stmmac])
[] (tnk_check_alive_timer+0x20/0xcc [stmmac]) from [] (call_timer_fn.isra.30+0x24/0x88)
[] (call_timer_fn.isra.30+0x24/0x88) from [] (run_timer_softirq+0x194/0x210)
[] (run_timer_softirq+0x194/0x210) from [] (__do_softirq+0x124/0x204)
[] (__do_softirq+0x124/0x204) from [] (irq_exit+0x9c/0xd0)
[] (irq_exit+0x9c/0xd0) from [] (handle_IRQ+0x44/0x90)
[] (handle_IRQ+0x44/0x90) from [] (gic_handle_irq+0x34/0x8c)
[] (gic_handle_irq+0x34/0x8c) from [] (__irq_svc+0x40/0x50)
Exception stack(0xd907bf98 to 0xd907bfe0)
bf80:                                                       c109c930 00000000
bfa0: 05d9fb66 00000000 c083a2a5 c0809d50 c05dee6c d907a000 c083a2a5 d907a000
bfc0: d907a000 c0809cf0 00000000 d907bfe0 c00142a4 c00142a8 60000013 ffffffff
[] (__irq_svc+0x40/0x50) from [] (arch_cpu_idle+0x2c/0x30)
[] (arch_cpu_idle+0x2c/0x30) from [] (cpu_startup_entry+0xfc/0x140)
[] (cpu_startup_entry+0xfc/0x140) from [<405d03e4>] (0x405d03e4)
SYS_GetLocalCurPts hi_sched_clock error u64TimeNow=244900145529808,u64TimeLast=244934488595280
~ #