Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

traceloop does not work with --host #2772

Closed
eiffel-fl opened this issue Apr 25, 2024 · 0 comments · Fixed by #2879
Closed

traceloop does not work with --host #2772

eiffel-fl opened this issue Apr 25, 2024 · 0 comments · Fixed by #2879
Assignees

Comments

@eiffel-fl
Copy link
Member

$ sudo ./ig traceloop --host                                          (tags/v0.18.0) %
CPU PID        COMM             SYSCALL                      PARAMS                                                                 RET  
^C7   91773      apache2          wait4                        upid=4294967295, stat_addr=0x7fffd079941c, options=3, ru=0x0           0
...

containers = append(containers, &containercollection.Container{Pid: 1})

740976d

> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).AttachContainer() ./pkg/gadgets/traceloop/tracer/tracer.go:698 (hits goroutine(1):4 total:4) (PC: 0x327624e)
   693:                 panic("event handler invalid")
   694:         }
   695:         t.eventCallback = nh
   696: }
   697:
=> 698: func (t *Tracer) AttachContainer(container *containercollection.Container) error {
   699:         t.waitGroup.Add(1)
   700:         err := t.Attach(container.Runtime.ContainerID, container.Mntns)
   701:         if err != nil {
   702:                 t.waitGroup.Done()
   703:                 return err
(dlv) p container
("*github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection.Container")(0xc002ff69a0)
*github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection.Container {
        Runtime: github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection.RuntimeMetadata {
                BasicRuntimeMetadata: (*"github.com/inspektor-gadget/inspektor-gadget/pkg/types.BasicRuntimeMetadata")(0xc002ff69a0),},
        K8s: github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection.K8sMetadata {
                BasicK8sMetadata: (*"github.com/inspektor-gadget/inspektor-gadget/pkg/types.BasicK8sMetadata")(0xc002ff69f0),
                PodUID: "",
                ownerReference: *k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference nil,},
        Pid: 1,
...
(dlv) n
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).AttachContainer() ./pkg/gadgets/traceloop/tracer/tracer.go:699 (PC: 0x3276268)
   694:         }
   695:         t.eventCallback = nh
   696: }
   697:
   698: func (t *Tracer) AttachContainer(container *containercollection.Container) error {
=> 699:         t.waitGroup.Add(1)
   700:         err := t.Attach(container.Runtime.ContainerID, container.Mntns)
   701:         if err != nil {
   702:                 t.waitGroup.Done()
   703:                 return err
   704:         }
(dlv) p container.Runtime.ContainerID
""
(dlv) p container.Mntns
0
# Attach is OK though and the perf buffer should be created
(dlv) b t.Read
(dlv) c
# Other terminal
$ sudo pkill -INT ig
(dlv) c
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(81):1 total:6) (PC: 0x3271296)
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(64):1 total:6) (PC: 0x3271296)
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(25):1 total:6) (PC: 0x3271296)
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(99):1 total:6) (PC: 0x3271296)
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(98):1 total:6) (PC: 0x3271296)
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Read() ./pkg/gadgets/traceloop/tracer/tracer.go:331 (hits goroutine(82):1 total:6) (PC: 0x3271296)
   326:                 return fmt.Sprintf("-1 (%s)", syscall.Errno(-errNo).Error())
   327:         }
   328:         return fmt.Sprintf("%d", ret)
   329: }
   330:
=> 331: func (t *Tracer) Read(containerID string) ([]*types.Event, error) {
(dlv) s
> github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/traceloop/tracer.(*Tracer).Attach() ./pkg/gadgets/traceloop/tracer/tracer.go:220 (PC: 0x3270756)
   215:         })
   216:
   217:         t.objs.Close()
   218: }
   219:
=> 220: func (t *Tracer) Attach(containerID string, mntnsID uint64) error {
   221:         innerBufferSpec := t.innerMapSpec.Copy()
   222:         innerBufferSpec.Name = fmt.Sprintf("perf_buffer_%d", mntnsID)
   223:
   224:         // 1. Create inner Map as perf buffer.
   225:         innerBuffer, err := ebpf.NewMap(innerBufferSpec)
(dlv) p mntnsID
0

But in the eBPF code, we use the real mntNsID:

perf_buffer = bpf_map_lookup_elem(&map_of_perf_buffers, &mntns_id);
if (!perf_buffer)
return 0;

$ git diff
diff --git a/pkg/gadgets/traceloop/tracer/bpf/traceloop.bpf.c b/pkg/gadgets/traceloop/tracer/bpf/traceloop.bpf.c
index 66cc9b41..80a2a093 100644
--- a/pkg/gadgets/traceloop/tracer/bpf/traceloop.bpf.c
+++ b/pkg/gadgets/traceloop/tracer/bpf/traceloop.bpf.c
@@ -171,6 +171,8 @@ int ig_traceloop_e(struct bpf_raw_tracepoint_args *ctx)
        int ret;
        int i;
 
+//     bpf_debug_printk("Syscall %d called", nr);
+
        if (should_filter_out_syscall(nr))
                return 0;
 
@@ -213,8 +215,10 @@ int ig_traceloop_e(struct bpf_raw_tracepoint_args *ctx)
        mntns_id = (u64)BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum);
 
        perf_buffer = bpf_map_lookup_elem(&map_of_perf_buffers, &mntns_id);
-       if (!perf_buffer)
+       if (!perf_buffer) {
+               bpf_error_printk("no perf buffer for mntns_id %d", mntns_id)
                return 0;
+       }
 
        bpf_get_current_comm(sc.comm, sizeof(sc.comm));
 
diff --git a/pkg/gadgets/traceloop/tracer/traceloop_arm64_bpfel.o b/pkg/gadgets/traceloop/tracer/traceloop_arm64_bpfel.o
deleted file mode 100644
index bd010c2b..00000000
Binary files a/pkg/gadgets/traceloop/tracer/traceloop_arm64_bpfel.o and /dev/null differ
diff --git a/pkg/gadgets/traceloop/tracer/traceloop_x86_bpfel.o b/pkg/gadgets/traceloop/tracer/traceloop_x86_bpfel.o
index ad27b05f..1af6092d 100644
Binary files a/pkg/gadgets/traceloop/tracer/traceloop_x86_bpfel.o and b/pkg/gadgets/traceloop/tracer/traceloop_x86_bpfel.o differ
diff --git a/pkg/gadgets/traceloop/tracer/tracer.go b/pkg/gadgets/traceloop/tracer/tracer.go
index c819f94f..49b04292 100644
--- a/pkg/gadgets/traceloop/tracer/tracer.go
+++ b/pkg/gadgets/traceloop/tracer/tracer.go
@@ -41,7 +41,7 @@ import (
        "github.com/inspektor-gadget/inspektor-gadget/pkg/utils/syscalls"
 )
 
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type syscall_event_t -type syscall_event_cont_t -target ${TARGET} -cc clang -cflags ${CFLAGS} traceloop ./bpf/traceloop.bpf.c -- -I./bpf/
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type syscall_event_t -type syscall_event_cont_t -target ${TARGET} -cc clang -cflags ${CFLAGS} traceloop ./bpf/traceloop.bpf.c -- -DSHOW_ERROR -I./bpf/
 
 // These consts must match the content of traceloop.h.
 const (
$ sudo ./traceloop --host
...
$ sudo cat /sys/kernel/debug/tracing/trace_pipe
...
        kwin_x11-3774    [003] ...2. 11405.042401: bpf_trace_printk: no perf buffer for mntns_id -268435455
...

Solution: Add a fake container but with the good mntNsID, i.e. corresponding to PID 1.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
Status: 🆕 New
1 participant