[lttng-dev] Segmentation fault in babeltrace when restarting live stream

[Anders Wallin]

Hi,

babeltrace seg faults when a live stream is restarted/recreated while
data is transferred. The test script can be used to run lttng on a
remote target, but I get the segfault on the same PC as well


Core was generated by `babeltrace -v -d -i lttng-live
net://localhost/host/arn-awallin-mint-l3/livetes'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00002af0e6d96182 in append_metadata (ctx=ctx at entry=0x1d63e90,
viewer_stream=viewer_stream at entry=0x1d6ef70) at lttng-live-comm.c:619
619             ret = ctf_append_trace_metadata(
(gdb) bt
#0  0x00002af0e6d96182 in append_metadata (ctx=ctx at entry=0x1d63e90,
viewer_stream=viewer_stream at entry=0x1d6ef70) at lttng-live-comm.c:619
#1  0x00002af0e6d986cb in get_next_index (stream_id=<synthetic
pointer>, index=0x1da2d10, viewer_stream=0x1d6ef70, ctx=0x1d63e90) at
lttng-live-comm.c:1029
#2  ctf_live_packet_seek (stream_pos=0x1da2208, index=<optimized out>,
whence=<optimized out>) at lttng-live-comm.c:1205
#3  0x00002af0e65589ee in prepare_mmap_stream_definition
(td=<optimized out>, packet_seek=0x2af0e6d97730
<ctf_live_packet_seek>, file_stream=0x1da1150) at ctf.c:2266
#4  ctf_open_mmap_stream_read (packet_seek=0x2af0e6d97730
<ctf_live_packet_seek>, mmap_info=0x1d70370, td=0x1d71490) at
ctf.c:2308
#5  ctf_open_mmap_trace_read (metadata_fp=<optimized out>,
packet_seek=0x2af0e6d97730 <ctf_live_packet_seek>,
mmap_list=0x7fffdabd0520, td=0x1d71490) at ctf.c:2359
#6  ctf_open_mmap_trace (mmap_list=0x7fffdabd0520,
packet_seek=0x2af0e6d97730 <ctf_live_packet_seek>,
metadata_fp=<optimized out>) at ctf.c:2393
#7  0x00002af0e6346694 in bt_context_add_trace
(ctx=ctx at entry=0x1d64f60, path=path at entry=0x0,
format_name=format_name at entry=0x2af0e6d99381 "ctf",
    packet_seek=packet_seek at entry=0x2af0e6d97730
<ctf_live_packet_seek>, stream_list=stream_list at entry=0x7fffdabd0520,
metadata=0x1d70250) at context.c:103
#8  0x00002af0e6d95ffb in add_one_trace (trace=0x1d70000,
ctx=0x1d63e90) at lttng-live-comm.c:1436
#9  add_traces (ctx=ctx at entry=0x1d63e90) at lttng-live-comm.c:1477
#10 0x00002af0e6d98daa in lttng_live_read (ctx=ctx at entry=0x1d63e90) at
lttng-live-comm.c:1695
#11 0x00002af0e6d95618 in lttng_live_open_trace_read (path=0x1d624b0
"net://localhost/host/arn-awallin-mint-l3/livetest") at
lttng-live-plugin.c:257
#12 lttng_live_open_trace (path=0x1d624b0
"net://localhost/host/arn-awallin-mint-l3/livetest", flags=<optimized
out>, packet_seek=<optimized out>, metadata_fp=<optimized out>)
    at lttng-live-plugin.c:295
#13 0x00002af0e634651d in bt_context_add_trace
(ctx=ctx at entry=0x1d60ad0, path=path at entry=0x1d624b0
"net://localhost/host/arn-awallin-mint-l3/livetest",
    format_name=format_name at entry=0x1d60b10 "lttng-live",
packet_seek=packet_seek at entry=0x0, stream_list=stream_list at entry=0x0,
metadata=metadata at entry=0x0) at context.c:95
#14 0x0000000000403120 in bt_context_add_traces_recursive
(ctx=ctx at entry=0x1d60ad0, path=<optimized out>,
    path at entry=0x1d624b0
"net://localhost/host/arn-awallin-mint-l3/livetest",
format_str=0x1d60b10 "lttng-live", packet_seek=packet_seek at entry=0x0)
at babeltrace.c:507
#15 0x0000000000402273 in main (argc=<optimized out>, argv=<optimized
out>) at babeltrace.c:736
(gdb)




Simplified script:
----------------------------------
#!/bin/bash

[ -z $APP ] && APP=./tracetest
DEBUG="-v -d"

# local target
SERVER_IP=localhost
TARGET_IP=localhost
SESSION=livetest

BT_INDEX=0
DATE=$(date +%H_%M_%S)

babeltrace_pid=-1

prepare_target () {
    TARGET_NAME=$(echo $(hostname))
    ulimit -c unlimited
}

target_start () {
    lttng create $SESSION -U net://$SERVER_IP --live 1000000
    lttng enable-event -a -u
    lttng start
}

target_stop () {
    lttng stop
    lttng destroy $SESSION
}

target_restart () {
    target_stop
    target_start
}

start_trace () {
    target_start

    # start babeltrace if it's not running
    if [ "$(kill -0 $babeltrace_pid)" != "0" ]; then
echo STARTING BABELTRACE
babeltrace $DEBUG -i lttng-live
net://localhost/host/$TARGET_NAME/$SESSION 2>&1 | tee
babeltrace_$BT_INDEX.log &
babeltrace_pid=$!
BT_INDEX=$(($BT_INDEX+1))
    fi

    ./tracetest 1000 3
    sleep 1
    target_restart
    ./tracetest 1000 3
    target_stop
}

############################
prepare_target

lttng-relayd -v -L net://localhost:5344 &> lttng-relayd.log &
lttng_relayd_pid=$!

for i in $(seq 1 3); do
    start_trace
    if [ -e core ]; then
        echo core dumped - saving data!!
tar czvf $0-$DATE.tgz core babeltrace*.log lttng-relayd.log
break
    fi
done

kill $lttng_relayd_pid
-------------------------------------------------------------


I'm using the latest git commit from the stable-1.2 branch. I have
test cases etc If you want to rerun the tests

Anders Wallin