[lttng-dev] epoll_wait reboot kernel?

zhenyu.ren zhenyu.ren at aliyun.com
Tue Feb 25 01:01:30 EST 2014


>Which gcc version do you use ?
$gcc --version
gcc (GCC) 4.1.2 20080704 (Red Hat 4.1.2-46)

I disassembed lib_ring_buffer_write() and focused on how it calles lib_ring_buffer_do_copy(indeed inline_memcpy)

void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
                           struct lib_ring_buffer_ctx *ctx,
                           const void *src, size_t len)
{
        struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
        struct channel_backend *chanb = &ctx->chan->backend;
        size_t sbidx, index;
        size_t offset = ctx->buf_offset;
        ssize_t pagecpy;
        struct lib_ring_buffer_backend_pages *rpages;
        unsigned long sb_bindex, id;
        if (unlikely(!len))
                return;
        offset &= chanb->buf_size - 1;
        sbidx = offset >> chanb->subbuf_size_order;
        index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
        pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
        id = bufb->buf_wsb[sbidx].id;
        sb_bindex = subbuffer_id_get_index(config, id);
        rpages = bufb->array[sb_bindex];
        CHAN_WARN_ON(ctx->chan,
                     config->mode == RING_BUFFER_OVERWRITE
                     && subbuffer_id_is_noref(config, id));
        if (likely(pagecpy == len))
                lib_ring_buffer_do_copy(config,
                                        rpages->p[index].virt
                                            + (offset & ~PAGE_MASK),
                                        src, len);
        else
                _lib_ring_buffer_write(bufb, offset, src, len, 0);
        ctx->buf_offset += len;
}
00000000000003a0 <lib_ring_buffer_write>:
     3a0:       55                      push   %rbp
     3a1:       48 89 e5                mov    %rsp,%rbp
     3a4:       48 83 ec 40             sub    $0x40,%rsp
     3a8:       48 89 5d d8             mov    %rbx,0xffffffffffffffd8(%rbp)
     3ac:       4c 89 65 e0             mov    %r12,0xffffffffffffffe0(%rbp)
     3b0:       4c 89 6d e8             mov    %r13,0xffffffffffffffe8(%rbp)
     3b4:       4c 89 75 f0             mov    %r14,0xfffffffffffffff0(%rbp)
     3b8:       4c 89 7d f8             mov    %r15,0xfffffffffffffff8(%rbp)
     3bc:       e8 00 00 00 00          callq  3c1 <lib_ring_buffer_write+0x21>
     3c1:       49 89 f5                mov    %rsi,%r13                                rsi/r13->               ctx
     3c4:       48 85 c9                test   %rcx,%rcx                                rcx->                   len
     3c7:       48 89 55 c8             mov    %rdx,0xffffffffffffffc8(%rbp)            rdx/c8->                src
     3cb:       49 89 cc                mov    %rcx,%r12                                rcx/r12->               len
     3ce:       48 8b 76 20             mov    0x20(%rsi),%rsi                          rsi->                   ctx->buf
     3d2:       4d 8b 45 00             mov    0x0(%r13),%r8                            r8->                    ctx->chan
     3d6:       49 8b 55 30             mov    0x30(%r13),%rdx                          rdx->                   offset = ctx->buf_offset
     3da:       0f 84 a5 00 00 00       je     485 <lib_ring_buffer_write+0xe5>         if(!len)  return
     3e0:       49 8b 58 10             mov    0x10(%r8),%rbx                           rbx->                   ctx->chan->backend
     3e4:       49 8d 40 10             lea    0x10(%r8),%rax                           rax->                   *chanb = &ctx->chan->backend
     3e8:       4c 8d 76 28             lea    0x28(%rsi),%r14                          r14->                   *bufb = &ctx->buf->backend
     3ec:       48 83 eb 01             sub    $0x1,%rbx                                rbx->                   chanb->buf_size - 1
     3f0:       48 21 d3                and    %rdx,%rbx                                rbx->                   offset &= chanb->buf_size -1
     3f3:       48 8b 50 08             mov    0x8(%rax),%rdx                           rdx->                   chanb->subbuf_size
     3f7:       48 89 55 d0             mov    %rdx,0xffffffffffffffd0(%rbp)            d0->                    chanb->subbuf_size
     3fb:       8b 48 10                mov    0x10(%rax),%ecx                          ecx->                   chanb->subbuf_size_order
     3fe:       48 89 da                mov    %rbx,%rdx                                rdx->                   offset
     401:       48 8b 46 28             mov    0x28(%rsi),%rax                          rax->                   bufb = &ctx->buf->backen
     405:       48 d3 ea                shr    %cl,%rdx                                 rdx->                   sbidx = offset >> chanb->subbuf_size_orde
     408:       48 8b 0c d0             mov    (%rax,%rdx,8),%rcx                       rcx->                   id = bufb->buf_wsb[sbidx].id
     40c:       8b 47 08                mov    0x8(%rdi),%eax                           eax->                   config->mode
     40f:       85 c0                   test   %eax,%eax                                CHAN_WARN_ON(... (config->mode == RING_BUFFER_OVERWRITE)...)
     411:       0f 85 89 00 00 00       jne    4a0 <lib_ring_buffer_write+0x100>
     417:       49 8b 46 10             mov    0x10(%r14),%rax                          rax->                   bufb->array
     41b:       48 89 ca                mov    %rcx,%rdx                                rdx->                   id
     41e:       83 e2 ff                and    $0xffffffffffffffff,%edx
     421:       4c 8b 3c d0             mov    (%rax,%rdx,8),%r15                       r15->                   rpages = bufb->array[sb_binde
     425:       48 b8 00 00 00 00 01    mov    $0x100000000,%rax                        rax->                   SB_ID_NOREF_MASK
     42c:       00 00 00
     42f:       48 85 c8                test   %rcx,%rax                                id & SB_ID_NOREF_MASK
     432:       0f 95 c0                setne  %al
     435:       0f b6 c0                movzbl %al,%eax
     438:       48 85 c0                test   %rax,%rax                                if(config->mode==RING_BUFFER_OVERWRITE && subbuffer_id_is_noref(config,id))
     43b:       75 6f                   jne    4ac <lib_ring_buffer_write+0x10c>
     43d:       48 89 d8                mov    %rbx,%rax                                rax->                   offset
     440:       48 f7 d8                neg    %rax                                     rax->                   -offset
     443:       25 ff 0f 00 00          and    $0xfff,%eax                              eax->                   (-offset) & ~PAGE_MASK
     448:       49 39 c4                cmp    %rax,%r12                                cmp(len,(-offset) & ~PAGE_MASK)
     44b:       49 0f 46 c4             cmovbe %r12,%rax                                rax->                   pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK)
     44f:       4c 39 e0                cmp    %r12,%rax                                if(pagecpy == len)
     452:       75 72                   jne    4c6 <lib_ring_buffer_write+0x126>        jmp to call _lib_ring_buffer_write()
     454:       48 8b 45 d0             mov    0xffffffffffffffd0(%rbp),%rax            rax->                   chanb->subbuf_size
     458:       48 8b 75 c8             mov    0xffffffffffffffc8(%rbp),%rsi            rsi->                   src
     45c:       48 83 e8 01             sub    $0x1,%rax                                rax->                   chanb->subbuf_size - 1
     460:       48 21 d8                and    %rbx,%rax                                rax->                   offset & (chanb->subbuf_size - 1)
     463:       81 e3 ff 0f 00 00       and    $0xfff,%ebx                              ebx->                   offset & ~PAGE_MASK
     469:       48 c1 e8 0c             shr    $0xc,%rax                                rax->                   index=(offset&(chanb->subbuf_size-1))>>PAGE_SHIFT
     46d:       48 c1 e0 04             shl    $0x4,%rax                                rax->                   index
     471:       4a 03 5c 38 20          add    0x20(%rax,%r15,1),%rbx                   rbx->                   rpages->p[index].virt+ (offset & ~PAGE_MASK)
     476:       48 89 da                mov    %rbx,%rdx                                rdx->                   SHOULD BE inline_memcpy() the 3rd parameter:len
     479:       48 89 df                mov    %rbx,%rdi                                rdi->                   1st parameter
     47c:       e8 00 00 00 00          callq  481 <lib_ring_buffer_write+0xe1>         call inline_memcpy(dst,src,len)
     481:       4d 01 65 30             add    %r12,0x30(%r13)                          ctx->buf_offset += len(r12)
     485:       48 8b 5d d8             mov    0xffffffffffffffd8(%rbp),%rbx
     489:       4c 8b 65 e0             mov    0xffffffffffffffe0(%rbp),%r12
     48d:       4c 8b 6d e8             mov    0xffffffffffffffe8(%rbp),%r13
     491:       4c 8b 75 f0             mov    0xfffffffffffffff0(%rbp),%r14
     495:       4c 8b 7d f8             mov    0xfffffffffffffff8(%rbp),%r15
     499:       c9                      leaveq
     49a:       c3                      retq
     49b:       0f 1f 44 00 00          nopl   0x0(%rax,%rax,1)
     4a0:       49 8b 46 10             mov    0x10(%r14),%rax                          rax->                   bufb->array
     4a4:       4c 8b 3c c8             mov    (%rax,%rcx,8),%r15                       r15->                   rpages = bufb->array[sb_bindex]
     4a8:       31 c0                   xor    %eax,%eax                                eax->                   0
     4aa:       eb 8c                   jmp    438 <lib_ring_buffer_write+0x98>         .......
     4ac:       f0 41 ff 00             lock incl (%r8)                                 atomic_inc(&__chan->record_disabled
     4b0:       be 6e 00 00 00          mov    $0x6e,%esi
     4b5:       48 c7 c7 00 00 00 00    mov    $0x0,%rdi
     4bc:       e8 00 00 00 00          callq  4c1 <lib_ring_buffer_write+0x121>        WARN_ON(1);
     4c1:       e9 77 ff ff ff          jmpq   43d <lib_ring_buffer_write+0x9d>         ....... jmp to if (pagecpy == len)
     4c6:       48 8b 55 c8             mov    0xffffffffffffffc8(%rbp),%rdx            rdx->                   src
     4ca:       45 31 c0                xor    %r8d,%r8d                                r8d->                   0
     4cd:       4c 89 e1                mov    %r12,%rcx                                r12/rcx->               len
     4d0:       48 89 de                mov    %rbx,%rsi                                rbx/rsi->               offset
     4d3:       4c 89 f7                mov    %r14,%rdi                                r14/rdi->               bufb
     4d6:       e8 00 00 00 00          callq  4db <lib_ring_buffer_write+0x13b>        _lib_ring_buffer_write(bufb, offset, src, len, 0);
     4db:       eb a4                   jmp    481 <lib_ring_buffer_write+0xe1>         ...... jmp to ctx->buf_offset += len and return


     All difference with good case is also where I am insterested in:

bad case:
     45c:       48 83 e8 01             sub    $0x1,%rax                                rax->                   chanb->subbuf_size - 1
     460:       48 21 d8                and    %rbx,%rax                                rax->                   offset & (chanb->subbuf_size - 1)
     463:       81 e3 ff 0f 00 00       and    $0xfff,%ebx                              ebx->                   offset & ~PAGE_MASK
     469:       48 c1 e8 0c             shr    $0xc,%rax                                rax->                   index=(offset&(chanb->subbuf_size-1))>>PAGE_SHIFT
     46d:       48 c1 e0 04             shl    $0x4,%rax                                rax->                   index
     471:       4a 03 5c 38 20          add    0x20(%rax,%r15,1),%rbx                   rbx->                   rpages->p[index].virt+ (offset & ~PAGE_MASK)
     476:       48 89 da                mov    %rbx,%rdx                                rdx->                   SHOULD BE inline_memcpy() the 3rd parameter:len
     479:       48 89 df                mov    %rbx,%rdi                                rdi->                   1st parameter
     47c:       e8 00 00 00 00          callq  481 <lib_ring_buffer_write+0xe1>         call inline_memcpy(dst,src,len)


good case:
     45c:       4c 89 e2                mov    %r12,%rdx                                r12/rdx->       len
     45f:       48 83 e8 01             sub    $0x1,%rax                                rax->           chanb->subbuf_size - 1
     463:       48 21 d8                and    %rbx,%rax                                rax->           offset & (chanb->subbuf_size - 1)
     466:       81 e3 ff 0f 00 00       and    $0xfff,%ebx                              ebx->           offset & ~PAGE_MASK
     46c:       48 c1 e8 0c             shr    $0xc,%rax                                rax->           index=(offset&(chanb->subbuf_size-1))>>PAGE_SHIFT
     470:       48 c1 e0 04             shl    $0x4,%rax                                rax->           index
     474:       4a 03 5c 38 20          add    0x20(%rax,%r15,1),%rbx                   rbx->           rpages->p[index].virt+ (offset & ~PAGE_MASK)
     479:       48 89 df                mov    %rbx,%rdi                                rdi->           dst , the 1st parameter of inline_memcpy(dst,src,len)
     47c:       e8 00 00 00 00          callq  481 <lib_ring_buffer_write+0xe1>         call inline_memcpy(dst,src,len)

In good case, at line 45c, rdx ,which is the 3rd parameter of inline_memcpy(dst,src,len), comes from  %r12 that is len.However,in bad case,at line 476, rdx comes from rbx that is the 1st parameter of inline_memcpy(dst,src,len). So you can image what will happen if len is too large.

I don't know why GCC produce the wrong code.Does anyone has any clue?

Thanks
zhenyu.ren


----- Original Message -----
> From: "zhenyu.ren" <zhenyu.ren at aliyun.com>
> To: "lttng-dev" <lttng-dev at lists.lttng.org>, "zhenyu.ren" <zhenyu.ren at aliyun.com>
> Sent: Monday, February 24, 2014 4:36:36 AM
> Subject: Re: [lttng-dev] epoll_wait reboot kernel?
> 
> Oh,It seems not epoll related . I am using DBG/ERR() as debug way ,but this
> print-style method doesn't work as expected,so I mistake for epoll_wait.
> It seems gcc problem because if  I changed lib_ring_buffer_do_copy as the
> following,all  work well
> 
> #define lib_ring_buffer_do_copy(config, dest, src, len)         \
> do {                                                            \
>         size_t __len1 = (len);                                  \
>         /*here , not __len=(len)*/
> mdelay(0x135);\
>         if (__builtin_constant_p(len))                          \
>                 memcpy(dest, src, __len1);                      \
>         else                                                    \
>                 inline_memcpy(dest, src, __len1);               \
> } while (0)
> 
> I disasembled lib_ring_buffer_write() and "vimdiff bad good" shows :
> 49c:       48 89 da                mov    %rbx,%rdx
> |       49c:
> 4c 89 f2                mov    %r14,%rdx
> 
> here rdx is the 3rd parameter to inline_memcpy().
> 
> Thanks
> zhenyu.ren
> 
> 
> >Hi,lttng-dev
> >
>  >  I am working on rhel6(2.6.32.220) and ust is working very well.But when I
>  >  use lttng-modules (after "lttng start"),the kernel reboots immediately
>  >  >without any message output to console.
> >
> >  It took me some days finding out what results in reboot. I shortcut
> >  __event_probe__##_name() before _tstruct but kernel still reboot(So what
> >  left is  >metadata_printf).If I comment out "mask |= POLLIN" in
> >  lttng_metadata_ring_buffer_poll() ,kernel do not reboot. What I find out
> >  is before reboot the >callchain of  lttng-consumerd thread is
> >  consumer_thread_metadata_poll()->lttng_poll_wait(wait for metadata
> >  stream)->compat_epoll_wait().
> >
> > I want to use poll instead as epoll in lttng-tools ,I changed lttng-tools
> > configure file to not use epoll(undef HAVE_EPOLL),but "lttng enable-event
> > -a -k >"failed :(Do I miss anything?)
> >#lttng enable-event -a -k
> >DEBUG1 [17837/18080]: Wait for client response (in thread_manage_clients()
> >at main.c:3709)
> >DEBUG1 [17837/18080]: Receiving data from client ... (in
> >thread_manage_clients() at main.c:3754)
> >DEBUG1 [17837/18080]: Nothing recv() from client... continuing (in
> >thread_manage_clients() at main.c:3758)
> >DEBUG1 [17837/18080]: Clean command context structure (in
> >clean_command_ctx() at main.c:535)
> >DEBUG1 [17837/18080]: Accepting client command ... (in
> >thread_manage_clients() at main.c:3667)
> >DEBUG1 [17837/18080]: Wait for client response (in thread_manage_clients()
> >at main.c:3709)
> >DEBUG1 [17837/18080]: Receiving data from client ... (in
> >thread_manage_clients() at main.c:3754)
> >DEBUG1 [17837/18080]: Processing client command 7 (in process_client_msg()
> >at main.c:2579)
> >DEBUG1 [17837/18080]: Getting session k-test by name (in
> >process_client_msg() at main.c:2656)
> >DEBUG1 [17837/18080]: Creating kernel session (in create_kernel_session() at
> >main.c:2495)
> >DEBUG3 [17837/18080]: Created hashtable size 4 at 0x669410 of type 1 (in
> >lttng_ht_new() at hashtable.c:112)
> >DEBUG1 [17837/18080]: Kernel session created (fd: 21) (in
> >kernel_create_session() at kernel.c:109)
> >DEBUG3 [17837/18080]: Copying tracing session consumer output in kernel
> >session (in copy_session_consumer() at main.c:2393)
> >DEBUG3 [17837/18080]: Created hashtable size 4 at 0x66a750 of type 1 (in
> >lttng_ht_new() at hashtable.c:112)
> >DEBUG3 [17837/18080]: Copy session consumer subdir /kernel (in
> >copy_session_consumer() at main.c:2428)
> >DEBUG3 [17837/18080]: mkdir() recursive
> >/root/lttng-traces/k-test-20140222-131335 with mode 504 for uid 0 and gid 0
> >(in run_as_mkdir_recursive() at >runas.c:310)
> >DEBUG1 [17837/18080]: Using run_as_clone (in run_as() at runas.c:293)
> >lttng-sessiond: ht-cleanup.c:92: thread_ht_cleanup: Assertion `pollfd ==
> >ht_cleanup_pipe[0]' failed.
> >
> >  It seems lttng-tools  in ubuntu 12.04 is using poll,isnt it?
> >
>  > Does anyone have any suggestions to me ?
> 
> >PS:.lttng(ust/module/tool) version is 2.3.0 and urcu version is 0.8
> 
> 
> _______________________________________________
> lttng-dev mailing list
> lttng-dev at lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> 

-- 
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com



More information about the lttng-dev mailing list