Merge branch '10GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/net...
[muen/linux.git] / tools / lib / bpf / libbpf.c
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4  * Common eBPF ELF object loading operations.
5  *
6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8  * Copyright (C) 2015 Huawei Inc.
9  * Copyright (C) 2017 Nicira, Inc.
10  */
11
12 #define _GNU_SOURCE
13 #include <stdlib.h>
14 #include <stdio.h>
15 #include <stdarg.h>
16 #include <libgen.h>
17 #include <inttypes.h>
18 #include <string.h>
19 #include <unistd.h>
20 #include <fcntl.h>
21 #include <errno.h>
22 #include <asm/unistd.h>
23 #include <linux/err.h>
24 #include <linux/kernel.h>
25 #include <linux/bpf.h>
26 #include <linux/btf.h>
27 #include <linux/list.h>
28 #include <linux/limits.h>
29 #include <linux/perf_event.h>
30 #include <linux/ring_buffer.h>
31 #include <sys/stat.h>
32 #include <sys/types.h>
33 #include <sys/vfs.h>
34 #include <tools/libc_compat.h>
35 #include <libelf.h>
36 #include <gelf.h>
37
38 #include "libbpf.h"
39 #include "bpf.h"
40 #include "btf.h"
41 #include "str_error.h"
42
43 #ifndef EM_BPF
44 #define EM_BPF 247
45 #endif
46
47 #ifndef BPF_FS_MAGIC
48 #define BPF_FS_MAGIC            0xcafe4a11
49 #endif
50
51 #define __printf(a, b)  __attribute__((format(printf, a, b)))
52
53 __printf(1, 2)
54 static int __base_pr(const char *format, ...)
55 {
56         va_list args;
57         int err;
58
59         va_start(args, format);
60         err = vfprintf(stderr, format, args);
61         va_end(args);
62         return err;
63 }
64
65 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
66 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
67 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
68
69 #define __pr(func, fmt, ...)    \
70 do {                            \
71         if ((func))             \
72                 (func)("libbpf: " fmt, ##__VA_ARGS__); \
73 } while (0)
74
75 #define pr_warning(fmt, ...)    __pr(__pr_warning, fmt, ##__VA_ARGS__)
76 #define pr_info(fmt, ...)       __pr(__pr_info, fmt, ##__VA_ARGS__)
77 #define pr_debug(fmt, ...)      __pr(__pr_debug, fmt, ##__VA_ARGS__)
78
79 void libbpf_set_print(libbpf_print_fn_t warn,
80                       libbpf_print_fn_t info,
81                       libbpf_print_fn_t debug)
82 {
83         __pr_warning = warn;
84         __pr_info = info;
85         __pr_debug = debug;
86 }
87
88 #define STRERR_BUFSIZE  128
89
90 #define CHECK_ERR(action, err, out) do {        \
91         err = action;                   \
92         if (err)                        \
93                 goto out;               \
94 } while(0)
95
96
97 /* Copied from tools/perf/util/util.h */
98 #ifndef zfree
99 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
100 #endif
101
102 #ifndef zclose
103 # define zclose(fd) ({                  \
104         int ___err = 0;                 \
105         if ((fd) >= 0)                  \
106                 ___err = close((fd));   \
107         fd = -1;                        \
108         ___err; })
109 #endif
110
111 #ifdef HAVE_LIBELF_MMAP_SUPPORT
112 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
113 #else
114 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
115 #endif
116
117 /*
118  * bpf_prog should be a better name but it has been used in
119  * linux/filter.h.
120  */
121 struct bpf_program {
122         /* Index in elf obj file, for relocation use. */
123         int idx;
124         char *name;
125         int prog_ifindex;
126         char *section_name;
127         struct bpf_insn *insns;
128         size_t insns_cnt, main_prog_cnt;
129         enum bpf_prog_type type;
130
131         struct reloc_desc {
132                 enum {
133                         RELO_LD64,
134                         RELO_CALL,
135                 } type;
136                 int insn_idx;
137                 union {
138                         int map_idx;
139                         int text_off;
140                 };
141         } *reloc_desc;
142         int nr_reloc;
143
144         struct {
145                 int nr;
146                 int *fds;
147         } instances;
148         bpf_program_prep_t preprocessor;
149
150         struct bpf_object *obj;
151         void *priv;
152         bpf_program_clear_priv_t clear_priv;
153
154         enum bpf_attach_type expected_attach_type;
155 };
156
157 struct bpf_map {
158         int fd;
159         char *name;
160         size_t offset;
161         int map_ifindex;
162         struct bpf_map_def def;
163         __u32 btf_key_type_id;
164         __u32 btf_value_type_id;
165         void *priv;
166         bpf_map_clear_priv_t clear_priv;
167 };
168
169 static LIST_HEAD(bpf_objects_list);
170
171 struct bpf_object {
172         char license[64];
173         __u32 kern_version;
174
175         struct bpf_program *programs;
176         size_t nr_programs;
177         struct bpf_map *maps;
178         size_t nr_maps;
179
180         bool loaded;
181         bool has_pseudo_calls;
182
183         /*
184          * Information when doing elf related work. Only valid if fd
185          * is valid.
186          */
187         struct {
188                 int fd;
189                 void *obj_buf;
190                 size_t obj_buf_sz;
191                 Elf *elf;
192                 GElf_Ehdr ehdr;
193                 Elf_Data *symbols;
194                 size_t strtabidx;
195                 struct {
196                         GElf_Shdr shdr;
197                         Elf_Data *data;
198                 } *reloc;
199                 int nr_reloc;
200                 int maps_shndx;
201                 int text_shndx;
202         } efile;
203         /*
204          * All loaded bpf_object is linked in a list, which is
205          * hidden to caller. bpf_objects__<func> handlers deal with
206          * all objects.
207          */
208         struct list_head list;
209
210         struct btf *btf;
211
212         void *priv;
213         bpf_object_clear_priv_t clear_priv;
214
215         char path[];
216 };
217 #define obj_elf_valid(o)        ((o)->efile.elf)
218
219 void bpf_program__unload(struct bpf_program *prog)
220 {
221         int i;
222
223         if (!prog)
224                 return;
225
226         /*
227          * If the object is opened but the program was never loaded,
228          * it is possible that prog->instances.nr == -1.
229          */
230         if (prog->instances.nr > 0) {
231                 for (i = 0; i < prog->instances.nr; i++)
232                         zclose(prog->instances.fds[i]);
233         } else if (prog->instances.nr != -1) {
234                 pr_warning("Internal error: instances.nr is %d\n",
235                            prog->instances.nr);
236         }
237
238         prog->instances.nr = -1;
239         zfree(&prog->instances.fds);
240 }
241
242 static void bpf_program__exit(struct bpf_program *prog)
243 {
244         if (!prog)
245                 return;
246
247         if (prog->clear_priv)
248                 prog->clear_priv(prog, prog->priv);
249
250         prog->priv = NULL;
251         prog->clear_priv = NULL;
252
253         bpf_program__unload(prog);
254         zfree(&prog->name);
255         zfree(&prog->section_name);
256         zfree(&prog->insns);
257         zfree(&prog->reloc_desc);
258
259         prog->nr_reloc = 0;
260         prog->insns_cnt = 0;
261         prog->idx = -1;
262 }
263
264 static int
265 bpf_program__init(void *data, size_t size, char *section_name, int idx,
266                   struct bpf_program *prog)
267 {
268         if (size < sizeof(struct bpf_insn)) {
269                 pr_warning("corrupted section '%s'\n", section_name);
270                 return -EINVAL;
271         }
272
273         bzero(prog, sizeof(*prog));
274
275         prog->section_name = strdup(section_name);
276         if (!prog->section_name) {
277                 pr_warning("failed to alloc name for prog under section(%d) %s\n",
278                            idx, section_name);
279                 goto errout;
280         }
281
282         prog->insns = malloc(size);
283         if (!prog->insns) {
284                 pr_warning("failed to alloc insns for prog under section %s\n",
285                            section_name);
286                 goto errout;
287         }
288         prog->insns_cnt = size / sizeof(struct bpf_insn);
289         memcpy(prog->insns, data,
290                prog->insns_cnt * sizeof(struct bpf_insn));
291         prog->idx = idx;
292         prog->instances.fds = NULL;
293         prog->instances.nr = -1;
294         prog->type = BPF_PROG_TYPE_KPROBE;
295
296         return 0;
297 errout:
298         bpf_program__exit(prog);
299         return -ENOMEM;
300 }
301
302 static int
303 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
304                         char *section_name, int idx)
305 {
306         struct bpf_program prog, *progs;
307         int nr_progs, err;
308
309         err = bpf_program__init(data, size, section_name, idx, &prog);
310         if (err)
311                 return err;
312
313         progs = obj->programs;
314         nr_progs = obj->nr_programs;
315
316         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
317         if (!progs) {
318                 /*
319                  * In this case the original obj->programs
320                  * is still valid, so don't need special treat for
321                  * bpf_close_object().
322                  */
323                 pr_warning("failed to alloc a new program under section '%s'\n",
324                            section_name);
325                 bpf_program__exit(&prog);
326                 return -ENOMEM;
327         }
328
329         pr_debug("found program %s\n", prog.section_name);
330         obj->programs = progs;
331         obj->nr_programs = nr_progs + 1;
332         prog.obj = obj;
333         progs[nr_progs] = prog;
334         return 0;
335 }
336
337 static int
338 bpf_object__init_prog_names(struct bpf_object *obj)
339 {
340         Elf_Data *symbols = obj->efile.symbols;
341         struct bpf_program *prog;
342         size_t pi, si;
343
344         for (pi = 0; pi < obj->nr_programs; pi++) {
345                 const char *name = NULL;
346
347                 prog = &obj->programs[pi];
348
349                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
350                      si++) {
351                         GElf_Sym sym;
352
353                         if (!gelf_getsym(symbols, si, &sym))
354                                 continue;
355                         if (sym.st_shndx != prog->idx)
356                                 continue;
357                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
358                                 continue;
359
360                         name = elf_strptr(obj->efile.elf,
361                                           obj->efile.strtabidx,
362                                           sym.st_name);
363                         if (!name) {
364                                 pr_warning("failed to get sym name string for prog %s\n",
365                                            prog->section_name);
366                                 return -LIBBPF_ERRNO__LIBELF;
367                         }
368                 }
369
370                 if (!name && prog->idx == obj->efile.text_shndx)
371                         name = ".text";
372
373                 if (!name) {
374                         pr_warning("failed to find sym for prog %s\n",
375                                    prog->section_name);
376                         return -EINVAL;
377                 }
378
379                 prog->name = strdup(name);
380                 if (!prog->name) {
381                         pr_warning("failed to allocate memory for prog sym %s\n",
382                                    name);
383                         return -ENOMEM;
384                 }
385         }
386
387         return 0;
388 }
389
390 static struct bpf_object *bpf_object__new(const char *path,
391                                           void *obj_buf,
392                                           size_t obj_buf_sz)
393 {
394         struct bpf_object *obj;
395
396         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
397         if (!obj) {
398                 pr_warning("alloc memory failed for %s\n", path);
399                 return ERR_PTR(-ENOMEM);
400         }
401
402         strcpy(obj->path, path);
403         obj->efile.fd = -1;
404
405         /*
406          * Caller of this function should also calls
407          * bpf_object__elf_finish() after data collection to return
408          * obj_buf to user. If not, we should duplicate the buffer to
409          * avoid user freeing them before elf finish.
410          */
411         obj->efile.obj_buf = obj_buf;
412         obj->efile.obj_buf_sz = obj_buf_sz;
413         obj->efile.maps_shndx = -1;
414
415         obj->loaded = false;
416
417         INIT_LIST_HEAD(&obj->list);
418         list_add(&obj->list, &bpf_objects_list);
419         return obj;
420 }
421
422 static void bpf_object__elf_finish(struct bpf_object *obj)
423 {
424         if (!obj_elf_valid(obj))
425                 return;
426
427         if (obj->efile.elf) {
428                 elf_end(obj->efile.elf);
429                 obj->efile.elf = NULL;
430         }
431         obj->efile.symbols = NULL;
432
433         zfree(&obj->efile.reloc);
434         obj->efile.nr_reloc = 0;
435         zclose(obj->efile.fd);
436         obj->efile.obj_buf = NULL;
437         obj->efile.obj_buf_sz = 0;
438 }
439
440 static int bpf_object__elf_init(struct bpf_object *obj)
441 {
442         int err = 0;
443         GElf_Ehdr *ep;
444
445         if (obj_elf_valid(obj)) {
446                 pr_warning("elf init: internal error\n");
447                 return -LIBBPF_ERRNO__LIBELF;
448         }
449
450         if (obj->efile.obj_buf_sz > 0) {
451                 /*
452                  * obj_buf should have been validated by
453                  * bpf_object__open_buffer().
454                  */
455                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
456                                             obj->efile.obj_buf_sz);
457         } else {
458                 obj->efile.fd = open(obj->path, O_RDONLY);
459                 if (obj->efile.fd < 0) {
460                         char errmsg[STRERR_BUFSIZE];
461                         char *cp = libbpf_strerror_r(errno, errmsg,
462                                                      sizeof(errmsg));
463
464                         pr_warning("failed to open %s: %s\n", obj->path, cp);
465                         return -errno;
466                 }
467
468                 obj->efile.elf = elf_begin(obj->efile.fd,
469                                 LIBBPF_ELF_C_READ_MMAP,
470                                 NULL);
471         }
472
473         if (!obj->efile.elf) {
474                 pr_warning("failed to open %s as ELF file\n",
475                                 obj->path);
476                 err = -LIBBPF_ERRNO__LIBELF;
477                 goto errout;
478         }
479
480         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
481                 pr_warning("failed to get EHDR from %s\n",
482                                 obj->path);
483                 err = -LIBBPF_ERRNO__FORMAT;
484                 goto errout;
485         }
486         ep = &obj->efile.ehdr;
487
488         /* Old LLVM set e_machine to EM_NONE */
489         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
490                 pr_warning("%s is not an eBPF object file\n",
491                         obj->path);
492                 err = -LIBBPF_ERRNO__FORMAT;
493                 goto errout;
494         }
495
496         return 0;
497 errout:
498         bpf_object__elf_finish(obj);
499         return err;
500 }
501
502 static int
503 bpf_object__check_endianness(struct bpf_object *obj)
504 {
505         static unsigned int const endian = 1;
506
507         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
508         case ELFDATA2LSB:
509                 /* We are big endian, BPF obj is little endian. */
510                 if (*(unsigned char const *)&endian != 1)
511                         goto mismatch;
512                 break;
513
514         case ELFDATA2MSB:
515                 /* We are little endian, BPF obj is big endian. */
516                 if (*(unsigned char const *)&endian != 0)
517                         goto mismatch;
518                 break;
519         default:
520                 return -LIBBPF_ERRNO__ENDIAN;
521         }
522
523         return 0;
524
525 mismatch:
526         pr_warning("Error: endianness mismatch.\n");
527         return -LIBBPF_ERRNO__ENDIAN;
528 }
529
530 static int
531 bpf_object__init_license(struct bpf_object *obj,
532                          void *data, size_t size)
533 {
534         memcpy(obj->license, data,
535                min(size, sizeof(obj->license) - 1));
536         pr_debug("license of %s is %s\n", obj->path, obj->license);
537         return 0;
538 }
539
540 static int
541 bpf_object__init_kversion(struct bpf_object *obj,
542                           void *data, size_t size)
543 {
544         __u32 kver;
545
546         if (size != sizeof(kver)) {
547                 pr_warning("invalid kver section in %s\n", obj->path);
548                 return -LIBBPF_ERRNO__FORMAT;
549         }
550         memcpy(&kver, data, sizeof(kver));
551         obj->kern_version = kver;
552         pr_debug("kernel version of %s is %x\n", obj->path,
553                  obj->kern_version);
554         return 0;
555 }
556
557 static int compare_bpf_map(const void *_a, const void *_b)
558 {
559         const struct bpf_map *a = _a;
560         const struct bpf_map *b = _b;
561
562         return a->offset - b->offset;
563 }
564
565 static int
566 bpf_object__init_maps(struct bpf_object *obj, int flags)
567 {
568         bool strict = !(flags & MAPS_RELAX_COMPAT);
569         int i, map_idx, map_def_sz, nr_maps = 0;
570         Elf_Scn *scn;
571         Elf_Data *data;
572         Elf_Data *symbols = obj->efile.symbols;
573
574         if (obj->efile.maps_shndx < 0)
575                 return -EINVAL;
576         if (!symbols)
577                 return -EINVAL;
578
579         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
580         if (scn)
581                 data = elf_getdata(scn, NULL);
582         if (!scn || !data) {
583                 pr_warning("failed to get Elf_Data from map section %d\n",
584                            obj->efile.maps_shndx);
585                 return -EINVAL;
586         }
587
588         /*
589          * Count number of maps. Each map has a name.
590          * Array of maps is not supported: only the first element is
591          * considered.
592          *
593          * TODO: Detect array of map and report error.
594          */
595         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
596                 GElf_Sym sym;
597
598                 if (!gelf_getsym(symbols, i, &sym))
599                         continue;
600                 if (sym.st_shndx != obj->efile.maps_shndx)
601                         continue;
602                 nr_maps++;
603         }
604
605         /* Alloc obj->maps and fill nr_maps. */
606         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
607                  nr_maps, data->d_size);
608
609         if (!nr_maps)
610                 return 0;
611
612         /* Assume equally sized map definitions */
613         map_def_sz = data->d_size / nr_maps;
614         if (!data->d_size || (data->d_size % nr_maps) != 0) {
615                 pr_warning("unable to determine map definition size "
616                            "section %s, %d maps in %zd bytes\n",
617                            obj->path, nr_maps, data->d_size);
618                 return -EINVAL;
619         }
620
621         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
622         if (!obj->maps) {
623                 pr_warning("alloc maps for object failed\n");
624                 return -ENOMEM;
625         }
626         obj->nr_maps = nr_maps;
627
628         /*
629          * fill all fd with -1 so won't close incorrect
630          * fd (fd=0 is stdin) when failure (zclose won't close
631          * negative fd)).
632          */
633         for (i = 0; i < nr_maps; i++)
634                 obj->maps[i].fd = -1;
635
636         /*
637          * Fill obj->maps using data in "maps" section.
638          */
639         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
640                 GElf_Sym sym;
641                 const char *map_name;
642                 struct bpf_map_def *def;
643
644                 if (!gelf_getsym(symbols, i, &sym))
645                         continue;
646                 if (sym.st_shndx != obj->efile.maps_shndx)
647                         continue;
648
649                 map_name = elf_strptr(obj->efile.elf,
650                                       obj->efile.strtabidx,
651                                       sym.st_name);
652                 obj->maps[map_idx].offset = sym.st_value;
653                 if (sym.st_value + map_def_sz > data->d_size) {
654                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
655                                    obj->path, map_name);
656                         return -EINVAL;
657                 }
658
659                 obj->maps[map_idx].name = strdup(map_name);
660                 if (!obj->maps[map_idx].name) {
661                         pr_warning("failed to alloc map name\n");
662                         return -ENOMEM;
663                 }
664                 pr_debug("map %d is \"%s\"\n", map_idx,
665                          obj->maps[map_idx].name);
666                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
667                 /*
668                  * If the definition of the map in the object file fits in
669                  * bpf_map_def, copy it.  Any extra fields in our version
670                  * of bpf_map_def will default to zero as a result of the
671                  * calloc above.
672                  */
673                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
674                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
675                 } else {
676                         /*
677                          * Here the map structure being read is bigger than what
678                          * we expect, truncate if the excess bits are all zero.
679                          * If they are not zero, reject this map as
680                          * incompatible.
681                          */
682                         char *b;
683                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
684                              b < ((char *)def) + map_def_sz; b++) {
685                                 if (*b != 0) {
686                                         pr_warning("maps section in %s: \"%s\" "
687                                                    "has unrecognized, non-zero "
688                                                    "options\n",
689                                                    obj->path, map_name);
690                                         if (strict)
691                                                 return -EINVAL;
692                                 }
693                         }
694                         memcpy(&obj->maps[map_idx].def, def,
695                                sizeof(struct bpf_map_def));
696                 }
697                 map_idx++;
698         }
699
700         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
701         return 0;
702 }
703
704 static bool section_have_execinstr(struct bpf_object *obj, int idx)
705 {
706         Elf_Scn *scn;
707         GElf_Shdr sh;
708
709         scn = elf_getscn(obj->efile.elf, idx);
710         if (!scn)
711                 return false;
712
713         if (gelf_getshdr(scn, &sh) != &sh)
714                 return false;
715
716         if (sh.sh_flags & SHF_EXECINSTR)
717                 return true;
718
719         return false;
720 }
721
722 static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
723 {
724         Elf *elf = obj->efile.elf;
725         GElf_Ehdr *ep = &obj->efile.ehdr;
726         Elf_Scn *scn = NULL;
727         int idx = 0, err = 0;
728
729         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
730         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
731                 pr_warning("failed to get e_shstrndx from %s\n",
732                            obj->path);
733                 return -LIBBPF_ERRNO__FORMAT;
734         }
735
736         while ((scn = elf_nextscn(elf, scn)) != NULL) {
737                 char *name;
738                 GElf_Shdr sh;
739                 Elf_Data *data;
740
741                 idx++;
742                 if (gelf_getshdr(scn, &sh) != &sh) {
743                         pr_warning("failed to get section(%d) header from %s\n",
744                                    idx, obj->path);
745                         err = -LIBBPF_ERRNO__FORMAT;
746                         goto out;
747                 }
748
749                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
750                 if (!name) {
751                         pr_warning("failed to get section(%d) name from %s\n",
752                                    idx, obj->path);
753                         err = -LIBBPF_ERRNO__FORMAT;
754                         goto out;
755                 }
756
757                 data = elf_getdata(scn, 0);
758                 if (!data) {
759                         pr_warning("failed to get section(%d) data from %s(%s)\n",
760                                    idx, name, obj->path);
761                         err = -LIBBPF_ERRNO__FORMAT;
762                         goto out;
763                 }
764                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
765                          idx, name, (unsigned long)data->d_size,
766                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
767                          (int)sh.sh_type);
768
769                 if (strcmp(name, "license") == 0)
770                         err = bpf_object__init_license(obj,
771                                                        data->d_buf,
772                                                        data->d_size);
773                 else if (strcmp(name, "version") == 0)
774                         err = bpf_object__init_kversion(obj,
775                                                         data->d_buf,
776                                                         data->d_size);
777                 else if (strcmp(name, "maps") == 0)
778                         obj->efile.maps_shndx = idx;
779                 else if (strcmp(name, BTF_ELF_SEC) == 0) {
780                         obj->btf = btf__new(data->d_buf, data->d_size,
781                                             __pr_debug);
782                         if (IS_ERR(obj->btf)) {
783                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
784                                            BTF_ELF_SEC, PTR_ERR(obj->btf));
785                                 obj->btf = NULL;
786                         }
787                 } else if (sh.sh_type == SHT_SYMTAB) {
788                         if (obj->efile.symbols) {
789                                 pr_warning("bpf: multiple SYMTAB in %s\n",
790                                            obj->path);
791                                 err = -LIBBPF_ERRNO__FORMAT;
792                         } else {
793                                 obj->efile.symbols = data;
794                                 obj->efile.strtabidx = sh.sh_link;
795                         }
796                 } else if ((sh.sh_type == SHT_PROGBITS) &&
797                            (sh.sh_flags & SHF_EXECINSTR) &&
798                            (data->d_size > 0)) {
799                         if (strcmp(name, ".text") == 0)
800                                 obj->efile.text_shndx = idx;
801                         err = bpf_object__add_program(obj, data->d_buf,
802                                                       data->d_size, name, idx);
803                         if (err) {
804                                 char errmsg[STRERR_BUFSIZE];
805                                 char *cp = libbpf_strerror_r(-err, errmsg,
806                                                              sizeof(errmsg));
807
808                                 pr_warning("failed to alloc program %s (%s): %s",
809                                            name, obj->path, cp);
810                         }
811                 } else if (sh.sh_type == SHT_REL) {
812                         void *reloc = obj->efile.reloc;
813                         int nr_reloc = obj->efile.nr_reloc + 1;
814                         int sec = sh.sh_info; /* points to other section */
815
816                         /* Only do relo for section with exec instructions */
817                         if (!section_have_execinstr(obj, sec)) {
818                                 pr_debug("skip relo %s(%d) for section(%d)\n",
819                                          name, idx, sec);
820                                 continue;
821                         }
822
823                         reloc = reallocarray(reloc, nr_reloc,
824                                              sizeof(*obj->efile.reloc));
825                         if (!reloc) {
826                                 pr_warning("realloc failed\n");
827                                 err = -ENOMEM;
828                         } else {
829                                 int n = nr_reloc - 1;
830
831                                 obj->efile.reloc = reloc;
832                                 obj->efile.nr_reloc = nr_reloc;
833
834                                 obj->efile.reloc[n].shdr = sh;
835                                 obj->efile.reloc[n].data = data;
836                         }
837                 } else {
838                         pr_debug("skip section(%d) %s\n", idx, name);
839                 }
840                 if (err)
841                         goto out;
842         }
843
844         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
845                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
846                 return LIBBPF_ERRNO__FORMAT;
847         }
848         if (obj->efile.maps_shndx >= 0) {
849                 err = bpf_object__init_maps(obj, flags);
850                 if (err)
851                         goto out;
852         }
853         err = bpf_object__init_prog_names(obj);
854 out:
855         return err;
856 }
857
858 static struct bpf_program *
859 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
860 {
861         struct bpf_program *prog;
862         size_t i;
863
864         for (i = 0; i < obj->nr_programs; i++) {
865                 prog = &obj->programs[i];
866                 if (prog->idx == idx)
867                         return prog;
868         }
869         return NULL;
870 }
871
872 struct bpf_program *
873 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
874 {
875         struct bpf_program *pos;
876
877         bpf_object__for_each_program(pos, obj) {
878                 if (pos->section_name && !strcmp(pos->section_name, title))
879                         return pos;
880         }
881         return NULL;
882 }
883
884 static int
885 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
886                            Elf_Data *data, struct bpf_object *obj)
887 {
888         Elf_Data *symbols = obj->efile.symbols;
889         int text_shndx = obj->efile.text_shndx;
890         int maps_shndx = obj->efile.maps_shndx;
891         struct bpf_map *maps = obj->maps;
892         size_t nr_maps = obj->nr_maps;
893         int i, nrels;
894
895         pr_debug("collecting relocating info for: '%s'\n",
896                  prog->section_name);
897         nrels = shdr->sh_size / shdr->sh_entsize;
898
899         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
900         if (!prog->reloc_desc) {
901                 pr_warning("failed to alloc memory in relocation\n");
902                 return -ENOMEM;
903         }
904         prog->nr_reloc = nrels;
905
906         for (i = 0; i < nrels; i++) {
907                 GElf_Sym sym;
908                 GElf_Rel rel;
909                 unsigned int insn_idx;
910                 struct bpf_insn *insns = prog->insns;
911                 size_t map_idx;
912
913                 if (!gelf_getrel(data, i, &rel)) {
914                         pr_warning("relocation: failed to get %d reloc\n", i);
915                         return -LIBBPF_ERRNO__FORMAT;
916                 }
917
918                 if (!gelf_getsym(symbols,
919                                  GELF_R_SYM(rel.r_info),
920                                  &sym)) {
921                         pr_warning("relocation: symbol %"PRIx64" not found\n",
922                                    GELF_R_SYM(rel.r_info));
923                         return -LIBBPF_ERRNO__FORMAT;
924                 }
925                 pr_debug("relo for %lld value %lld name %d\n",
926                          (long long) (rel.r_info >> 32),
927                          (long long) sym.st_value, sym.st_name);
928
929                 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
930                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
931                                    prog->section_name, sym.st_shndx);
932                         return -LIBBPF_ERRNO__RELOC;
933                 }
934
935                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
936                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
937
938                 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
939                         if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
940                                 pr_warning("incorrect bpf_call opcode\n");
941                                 return -LIBBPF_ERRNO__RELOC;
942                         }
943                         prog->reloc_desc[i].type = RELO_CALL;
944                         prog->reloc_desc[i].insn_idx = insn_idx;
945                         prog->reloc_desc[i].text_off = sym.st_value;
946                         obj->has_pseudo_calls = true;
947                         continue;
948                 }
949
950                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
951                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
952                                    insn_idx, insns[insn_idx].code);
953                         return -LIBBPF_ERRNO__RELOC;
954                 }
955
956                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
957                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
958                         if (maps[map_idx].offset == sym.st_value) {
959                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
960                                          map_idx, maps[map_idx].name, insn_idx);
961                                 break;
962                         }
963                 }
964
965                 if (map_idx >= nr_maps) {
966                         pr_warning("bpf relocation: map_idx %d large than %d\n",
967                                    (int)map_idx, (int)nr_maps - 1);
968                         return -LIBBPF_ERRNO__RELOC;
969                 }
970
971                 prog->reloc_desc[i].type = RELO_LD64;
972                 prog->reloc_desc[i].insn_idx = insn_idx;
973                 prog->reloc_desc[i].map_idx = map_idx;
974         }
975         return 0;
976 }
977
978 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
979 {
980         const struct btf_type *container_type;
981         const struct btf_member *key, *value;
982         struct bpf_map_def *def = &map->def;
983         const size_t max_name = 256;
984         char container_name[max_name];
985         __s64 key_size, value_size;
986         __s32 container_id;
987
988         if (snprintf(container_name, max_name, "____btf_map_%s", map->name) ==
989             max_name) {
990                 pr_warning("map:%s length of '____btf_map_%s' is too long\n",
991                            map->name, map->name);
992                 return -EINVAL;
993         }
994
995         container_id = btf__find_by_name(btf, container_name);
996         if (container_id < 0) {
997                 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
998                          map->name, container_name);
999                 return container_id;
1000         }
1001
1002         container_type = btf__type_by_id(btf, container_id);
1003         if (!container_type) {
1004                 pr_warning("map:%s cannot find BTF type for container_id:%u\n",
1005                            map->name, container_id);
1006                 return -EINVAL;
1007         }
1008
1009         if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT ||
1010             BTF_INFO_VLEN(container_type->info) < 2) {
1011                 pr_warning("map:%s container_name:%s is an invalid container struct\n",
1012                            map->name, container_name);
1013                 return -EINVAL;
1014         }
1015
1016         key = (struct btf_member *)(container_type + 1);
1017         value = key + 1;
1018
1019         key_size = btf__resolve_size(btf, key->type);
1020         if (key_size < 0) {
1021                 pr_warning("map:%s invalid BTF key_type_size\n",
1022                            map->name);
1023                 return key_size;
1024         }
1025
1026         if (def->key_size != key_size) {
1027                 pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1028                            map->name, (__u32)key_size, def->key_size);
1029                 return -EINVAL;
1030         }
1031
1032         value_size = btf__resolve_size(btf, value->type);
1033         if (value_size < 0) {
1034                 pr_warning("map:%s invalid BTF value_type_size\n", map->name);
1035                 return value_size;
1036         }
1037
1038         if (def->value_size != value_size) {
1039                 pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1040                            map->name, (__u32)value_size, def->value_size);
1041                 return -EINVAL;
1042         }
1043
1044         map->btf_key_type_id = key->type;
1045         map->btf_value_type_id = value->type;
1046
1047         return 0;
1048 }
1049
1050 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1051 {
1052         struct bpf_map_info info = {};
1053         __u32 len = sizeof(info);
1054         int new_fd, err;
1055         char *new_name;
1056
1057         err = bpf_obj_get_info_by_fd(fd, &info, &len);
1058         if (err)
1059                 return err;
1060
1061         new_name = strdup(info.name);
1062         if (!new_name)
1063                 return -errno;
1064
1065         new_fd = open("/", O_RDONLY | O_CLOEXEC);
1066         if (new_fd < 0)
1067                 goto err_free_new_name;
1068
1069         new_fd = dup3(fd, new_fd, O_CLOEXEC);
1070         if (new_fd < 0)
1071                 goto err_close_new_fd;
1072
1073         err = zclose(map->fd);
1074         if (err)
1075                 goto err_close_new_fd;
1076         free(map->name);
1077
1078         map->fd = new_fd;
1079         map->name = new_name;
1080         map->def.type = info.type;
1081         map->def.key_size = info.key_size;
1082         map->def.value_size = info.value_size;
1083         map->def.max_entries = info.max_entries;
1084         map->def.map_flags = info.map_flags;
1085         map->btf_key_type_id = info.btf_key_type_id;
1086         map->btf_value_type_id = info.btf_value_type_id;
1087
1088         return 0;
1089
1090 err_close_new_fd:
1091         close(new_fd);
1092 err_free_new_name:
1093         free(new_name);
1094         return -errno;
1095 }
1096
1097 static int
1098 bpf_object__create_maps(struct bpf_object *obj)
1099 {
1100         struct bpf_create_map_attr create_attr = {};
1101         unsigned int i;
1102         int err;
1103
1104         for (i = 0; i < obj->nr_maps; i++) {
1105                 struct bpf_map *map = &obj->maps[i];
1106                 struct bpf_map_def *def = &map->def;
1107                 char *cp, errmsg[STRERR_BUFSIZE];
1108                 int *pfd = &map->fd;
1109
1110                 if (map->fd >= 0) {
1111                         pr_debug("skip map create (preset) %s: fd=%d\n",
1112                                  map->name, map->fd);
1113                         continue;
1114                 }
1115
1116                 create_attr.name = map->name;
1117                 create_attr.map_ifindex = map->map_ifindex;
1118                 create_attr.map_type = def->type;
1119                 create_attr.map_flags = def->map_flags;
1120                 create_attr.key_size = def->key_size;
1121                 create_attr.value_size = def->value_size;
1122                 create_attr.max_entries = def->max_entries;
1123                 create_attr.btf_fd = 0;
1124                 create_attr.btf_key_type_id = 0;
1125                 create_attr.btf_value_type_id = 0;
1126
1127                 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1128                         create_attr.btf_fd = btf__fd(obj->btf);
1129                         create_attr.btf_key_type_id = map->btf_key_type_id;
1130                         create_attr.btf_value_type_id = map->btf_value_type_id;
1131                 }
1132
1133                 *pfd = bpf_create_map_xattr(&create_attr);
1134                 if (*pfd < 0 && create_attr.btf_key_type_id) {
1135                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1136                         pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1137                                    map->name, cp, errno);
1138                         create_attr.btf_fd = 0;
1139                         create_attr.btf_key_type_id = 0;
1140                         create_attr.btf_value_type_id = 0;
1141                         map->btf_key_type_id = 0;
1142                         map->btf_value_type_id = 0;
1143                         *pfd = bpf_create_map_xattr(&create_attr);
1144                 }
1145
1146                 if (*pfd < 0) {
1147                         size_t j;
1148
1149                         err = *pfd;
1150                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1151                         pr_warning("failed to create map (name: '%s'): %s\n",
1152                                    map->name, cp);
1153                         for (j = 0; j < i; j++)
1154                                 zclose(obj->maps[j].fd);
1155                         return err;
1156                 }
1157                 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1158         }
1159
1160         return 0;
1161 }
1162
1163 static int
1164 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1165                         struct reloc_desc *relo)
1166 {
1167         struct bpf_insn *insn, *new_insn;
1168         struct bpf_program *text;
1169         size_t new_cnt;
1170
1171         if (relo->type != RELO_CALL)
1172                 return -LIBBPF_ERRNO__RELOC;
1173
1174         if (prog->idx == obj->efile.text_shndx) {
1175                 pr_warning("relo in .text insn %d into off %d\n",
1176                            relo->insn_idx, relo->text_off);
1177                 return -LIBBPF_ERRNO__RELOC;
1178         }
1179
1180         if (prog->main_prog_cnt == 0) {
1181                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1182                 if (!text) {
1183                         pr_warning("no .text section found yet relo into text exist\n");
1184                         return -LIBBPF_ERRNO__RELOC;
1185                 }
1186                 new_cnt = prog->insns_cnt + text->insns_cnt;
1187                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
1188                 if (!new_insn) {
1189                         pr_warning("oom in prog realloc\n");
1190                         return -ENOMEM;
1191                 }
1192                 memcpy(new_insn + prog->insns_cnt, text->insns,
1193                        text->insns_cnt * sizeof(*insn));
1194                 prog->insns = new_insn;
1195                 prog->main_prog_cnt = prog->insns_cnt;
1196                 prog->insns_cnt = new_cnt;
1197                 pr_debug("added %zd insn from %s to prog %s\n",
1198                          text->insns_cnt, text->section_name,
1199                          prog->section_name);
1200         }
1201         insn = &prog->insns[relo->insn_idx];
1202         insn->imm += prog->main_prog_cnt - relo->insn_idx;
1203         return 0;
1204 }
1205
1206 static int
1207 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1208 {
1209         int i, err;
1210
1211         if (!prog || !prog->reloc_desc)
1212                 return 0;
1213
1214         for (i = 0; i < prog->nr_reloc; i++) {
1215                 if (prog->reloc_desc[i].type == RELO_LD64) {
1216                         struct bpf_insn *insns = prog->insns;
1217                         int insn_idx, map_idx;
1218
1219                         insn_idx = prog->reloc_desc[i].insn_idx;
1220                         map_idx = prog->reloc_desc[i].map_idx;
1221
1222                         if (insn_idx >= (int)prog->insns_cnt) {
1223                                 pr_warning("relocation out of range: '%s'\n",
1224                                            prog->section_name);
1225                                 return -LIBBPF_ERRNO__RELOC;
1226                         }
1227                         insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1228                         insns[insn_idx].imm = obj->maps[map_idx].fd;
1229                 } else {
1230                         err = bpf_program__reloc_text(prog, obj,
1231                                                       &prog->reloc_desc[i]);
1232                         if (err)
1233                                 return err;
1234                 }
1235         }
1236
1237         zfree(&prog->reloc_desc);
1238         prog->nr_reloc = 0;
1239         return 0;
1240 }
1241
1242
1243 static int
1244 bpf_object__relocate(struct bpf_object *obj)
1245 {
1246         struct bpf_program *prog;
1247         size_t i;
1248         int err;
1249
1250         for (i = 0; i < obj->nr_programs; i++) {
1251                 prog = &obj->programs[i];
1252
1253                 err = bpf_program__relocate(prog, obj);
1254                 if (err) {
1255                         pr_warning("failed to relocate '%s'\n",
1256                                    prog->section_name);
1257                         return err;
1258                 }
1259         }
1260         return 0;
1261 }
1262
1263 static int bpf_object__collect_reloc(struct bpf_object *obj)
1264 {
1265         int i, err;
1266
1267         if (!obj_elf_valid(obj)) {
1268                 pr_warning("Internal error: elf object is closed\n");
1269                 return -LIBBPF_ERRNO__INTERNAL;
1270         }
1271
1272         for (i = 0; i < obj->efile.nr_reloc; i++) {
1273                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1274                 Elf_Data *data = obj->efile.reloc[i].data;
1275                 int idx = shdr->sh_info;
1276                 struct bpf_program *prog;
1277
1278                 if (shdr->sh_type != SHT_REL) {
1279                         pr_warning("internal error at %d\n", __LINE__);
1280                         return -LIBBPF_ERRNO__INTERNAL;
1281                 }
1282
1283                 prog = bpf_object__find_prog_by_idx(obj, idx);
1284                 if (!prog) {
1285                         pr_warning("relocation failed: no section(%d)\n", idx);
1286                         return -LIBBPF_ERRNO__RELOC;
1287                 }
1288
1289                 err = bpf_program__collect_reloc(prog,
1290                                                  shdr, data,
1291                                                  obj);
1292                 if (err)
1293                         return err;
1294         }
1295         return 0;
1296 }
1297
1298 static int
1299 load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1300              const char *name, struct bpf_insn *insns, int insns_cnt,
1301              char *license, __u32 kern_version, int *pfd, int prog_ifindex)
1302 {
1303         struct bpf_load_program_attr load_attr;
1304         char *cp, errmsg[STRERR_BUFSIZE];
1305         char *log_buf;
1306         int ret;
1307
1308         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1309         load_attr.prog_type = type;
1310         load_attr.expected_attach_type = expected_attach_type;
1311         load_attr.name = name;
1312         load_attr.insns = insns;
1313         load_attr.insns_cnt = insns_cnt;
1314         load_attr.license = license;
1315         load_attr.kern_version = kern_version;
1316         load_attr.prog_ifindex = prog_ifindex;
1317
1318         if (!load_attr.insns || !load_attr.insns_cnt)
1319                 return -EINVAL;
1320
1321         log_buf = malloc(BPF_LOG_BUF_SIZE);
1322         if (!log_buf)
1323                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1324
1325         ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1326
1327         if (ret >= 0) {
1328                 *pfd = ret;
1329                 ret = 0;
1330                 goto out;
1331         }
1332
1333         ret = -LIBBPF_ERRNO__LOAD;
1334         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1335         pr_warning("load bpf program failed: %s\n", cp);
1336
1337         if (log_buf && log_buf[0] != '\0') {
1338                 ret = -LIBBPF_ERRNO__VERIFY;
1339                 pr_warning("-- BEGIN DUMP LOG ---\n");
1340                 pr_warning("\n%s\n", log_buf);
1341                 pr_warning("-- END LOG --\n");
1342         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1343                 pr_warning("Program too large (%zu insns), at most %d insns\n",
1344                            load_attr.insns_cnt, BPF_MAXINSNS);
1345                 ret = -LIBBPF_ERRNO__PROG2BIG;
1346         } else {
1347                 /* Wrong program type? */
1348                 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1349                         int fd;
1350
1351                         load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1352                         load_attr.expected_attach_type = 0;
1353                         fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1354                         if (fd >= 0) {
1355                                 close(fd);
1356                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1357                                 goto out;
1358                         }
1359                 }
1360
1361                 if (log_buf)
1362                         ret = -LIBBPF_ERRNO__KVER;
1363         }
1364
1365 out:
1366         free(log_buf);
1367         return ret;
1368 }
1369
1370 int
1371 bpf_program__load(struct bpf_program *prog,
1372                   char *license, __u32 kern_version)
1373 {
1374         int err = 0, fd, i;
1375
1376         if (prog->instances.nr < 0 || !prog->instances.fds) {
1377                 if (prog->preprocessor) {
1378                         pr_warning("Internal error: can't load program '%s'\n",
1379                                    prog->section_name);
1380                         return -LIBBPF_ERRNO__INTERNAL;
1381                 }
1382
1383                 prog->instances.fds = malloc(sizeof(int));
1384                 if (!prog->instances.fds) {
1385                         pr_warning("Not enough memory for BPF fds\n");
1386                         return -ENOMEM;
1387                 }
1388                 prog->instances.nr = 1;
1389                 prog->instances.fds[0] = -1;
1390         }
1391
1392         if (!prog->preprocessor) {
1393                 if (prog->instances.nr != 1) {
1394                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1395                                    prog->section_name, prog->instances.nr);
1396                 }
1397                 err = load_program(prog->type, prog->expected_attach_type,
1398                                    prog->name, prog->insns, prog->insns_cnt,
1399                                    license, kern_version, &fd,
1400                                    prog->prog_ifindex);
1401                 if (!err)
1402                         prog->instances.fds[0] = fd;
1403                 goto out;
1404         }
1405
1406         for (i = 0; i < prog->instances.nr; i++) {
1407                 struct bpf_prog_prep_result result;
1408                 bpf_program_prep_t preprocessor = prog->preprocessor;
1409
1410                 bzero(&result, sizeof(result));
1411                 err = preprocessor(prog, i, prog->insns,
1412                                    prog->insns_cnt, &result);
1413                 if (err) {
1414                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1415                                    i, prog->section_name);
1416                         goto out;
1417                 }
1418
1419                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1420                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1421                                  i, prog->section_name);
1422                         prog->instances.fds[i] = -1;
1423                         if (result.pfd)
1424                                 *result.pfd = -1;
1425                         continue;
1426                 }
1427
1428                 err = load_program(prog->type, prog->expected_attach_type,
1429                                    prog->name, result.new_insn_ptr,
1430                                    result.new_insn_cnt,
1431                                    license, kern_version, &fd,
1432                                    prog->prog_ifindex);
1433
1434                 if (err) {
1435                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1436                                         i, prog->section_name);
1437                         goto out;
1438                 }
1439
1440                 if (result.pfd)
1441                         *result.pfd = fd;
1442                 prog->instances.fds[i] = fd;
1443         }
1444 out:
1445         if (err)
1446                 pr_warning("failed to load program '%s'\n",
1447                            prog->section_name);
1448         zfree(&prog->insns);
1449         prog->insns_cnt = 0;
1450         return err;
1451 }
1452
1453 static bool bpf_program__is_function_storage(struct bpf_program *prog,
1454                                              struct bpf_object *obj)
1455 {
1456         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
1457 }
1458
1459 static int
1460 bpf_object__load_progs(struct bpf_object *obj)
1461 {
1462         size_t i;
1463         int err;
1464
1465         for (i = 0; i < obj->nr_programs; i++) {
1466                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
1467                         continue;
1468                 err = bpf_program__load(&obj->programs[i],
1469                                         obj->license,
1470                                         obj->kern_version);
1471                 if (err)
1472                         return err;
1473         }
1474         return 0;
1475 }
1476
1477 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
1478 {
1479         switch (type) {
1480         case BPF_PROG_TYPE_SOCKET_FILTER:
1481         case BPF_PROG_TYPE_SCHED_CLS:
1482         case BPF_PROG_TYPE_SCHED_ACT:
1483         case BPF_PROG_TYPE_XDP:
1484         case BPF_PROG_TYPE_CGROUP_SKB:
1485         case BPF_PROG_TYPE_CGROUP_SOCK:
1486         case BPF_PROG_TYPE_LWT_IN:
1487         case BPF_PROG_TYPE_LWT_OUT:
1488         case BPF_PROG_TYPE_LWT_XMIT:
1489         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1490         case BPF_PROG_TYPE_SOCK_OPS:
1491         case BPF_PROG_TYPE_SK_SKB:
1492         case BPF_PROG_TYPE_CGROUP_DEVICE:
1493         case BPF_PROG_TYPE_SK_MSG:
1494         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1495         case BPF_PROG_TYPE_LIRC_MODE2:
1496         case BPF_PROG_TYPE_SK_REUSEPORT:
1497         case BPF_PROG_TYPE_FLOW_DISSECTOR:
1498                 return false;
1499         case BPF_PROG_TYPE_UNSPEC:
1500         case BPF_PROG_TYPE_KPROBE:
1501         case BPF_PROG_TYPE_TRACEPOINT:
1502         case BPF_PROG_TYPE_PERF_EVENT:
1503         case BPF_PROG_TYPE_RAW_TRACEPOINT:
1504         default:
1505                 return true;
1506         }
1507 }
1508
1509 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
1510 {
1511         if (needs_kver && obj->kern_version == 0) {
1512                 pr_warning("%s doesn't provide kernel version\n",
1513                            obj->path);
1514                 return -LIBBPF_ERRNO__KVERSION;
1515         }
1516         return 0;
1517 }
1518
1519 static struct bpf_object *
1520 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
1521                    bool needs_kver, int flags)
1522 {
1523         struct bpf_object *obj;
1524         int err;
1525
1526         if (elf_version(EV_CURRENT) == EV_NONE) {
1527                 pr_warning("failed to init libelf for %s\n", path);
1528                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1529         }
1530
1531         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1532         if (IS_ERR(obj))
1533                 return obj;
1534
1535         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1536         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1537         CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);
1538         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1539         CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1540
1541         bpf_object__elf_finish(obj);
1542         return obj;
1543 out:
1544         bpf_object__close(obj);
1545         return ERR_PTR(err);
1546 }
1547
1548 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr,
1549                                             int flags)
1550 {
1551         /* param validation */
1552         if (!attr->file)
1553                 return NULL;
1554
1555         pr_debug("loading %s\n", attr->file);
1556
1557         return __bpf_object__open(attr->file, NULL, 0,
1558                                   bpf_prog_type__needs_kver(attr->prog_type),
1559                                   flags);
1560 }
1561
1562 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
1563 {
1564         return __bpf_object__open_xattr(attr, 0);
1565 }
1566
1567 struct bpf_object *bpf_object__open(const char *path)
1568 {
1569         struct bpf_object_open_attr attr = {
1570                 .file           = path,
1571                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
1572         };
1573
1574         return bpf_object__open_xattr(&attr);
1575 }
1576
1577 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1578                                            size_t obj_buf_sz,
1579                                            const char *name)
1580 {
1581         char tmp_name[64];
1582
1583         /* param validation */
1584         if (!obj_buf || obj_buf_sz <= 0)
1585                 return NULL;
1586
1587         if (!name) {
1588                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1589                          (unsigned long)obj_buf,
1590                          (unsigned long)obj_buf_sz);
1591                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1592                 name = tmp_name;
1593         }
1594         pr_debug("loading object '%s' from buffer\n",
1595                  name);
1596
1597         return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true);
1598 }
1599
1600 int bpf_object__unload(struct bpf_object *obj)
1601 {
1602         size_t i;
1603
1604         if (!obj)
1605                 return -EINVAL;
1606
1607         for (i = 0; i < obj->nr_maps; i++)
1608                 zclose(obj->maps[i].fd);
1609
1610         for (i = 0; i < obj->nr_programs; i++)
1611                 bpf_program__unload(&obj->programs[i]);
1612
1613         return 0;
1614 }
1615
1616 int bpf_object__load(struct bpf_object *obj)
1617 {
1618         int err;
1619
1620         if (!obj)
1621                 return -EINVAL;
1622
1623         if (obj->loaded) {
1624                 pr_warning("object should not be loaded twice\n");
1625                 return -EINVAL;
1626         }
1627
1628         obj->loaded = true;
1629
1630         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1631         CHECK_ERR(bpf_object__relocate(obj), err, out);
1632         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1633
1634         return 0;
1635 out:
1636         bpf_object__unload(obj);
1637         pr_warning("failed to load object '%s'\n", obj->path);
1638         return err;
1639 }
1640
1641 static int check_path(const char *path)
1642 {
1643         char *cp, errmsg[STRERR_BUFSIZE];
1644         struct statfs st_fs;
1645         char *dname, *dir;
1646         int err = 0;
1647
1648         if (path == NULL)
1649                 return -EINVAL;
1650
1651         dname = strdup(path);
1652         if (dname == NULL)
1653                 return -ENOMEM;
1654
1655         dir = dirname(dname);
1656         if (statfs(dir, &st_fs)) {
1657                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1658                 pr_warning("failed to statfs %s: %s\n", dir, cp);
1659                 err = -errno;
1660         }
1661         free(dname);
1662
1663         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1664                 pr_warning("specified path %s is not on BPF FS\n", path);
1665                 err = -EINVAL;
1666         }
1667
1668         return err;
1669 }
1670
1671 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1672                               int instance)
1673 {
1674         char *cp, errmsg[STRERR_BUFSIZE];
1675         int err;
1676
1677         err = check_path(path);
1678         if (err)
1679                 return err;
1680
1681         if (prog == NULL) {
1682                 pr_warning("invalid program pointer\n");
1683                 return -EINVAL;
1684         }
1685
1686         if (instance < 0 || instance >= prog->instances.nr) {
1687                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1688                            instance, prog->section_name, prog->instances.nr);
1689                 return -EINVAL;
1690         }
1691
1692         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1693                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1694                 pr_warning("failed to pin program: %s\n", cp);
1695                 return -errno;
1696         }
1697         pr_debug("pinned program '%s'\n", path);
1698
1699         return 0;
1700 }
1701
1702 static int make_dir(const char *path)
1703 {
1704         char *cp, errmsg[STRERR_BUFSIZE];
1705         int err = 0;
1706
1707         if (mkdir(path, 0700) && errno != EEXIST)
1708                 err = -errno;
1709
1710         if (err) {
1711                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
1712                 pr_warning("failed to mkdir %s: %s\n", path, cp);
1713         }
1714         return err;
1715 }
1716
1717 int bpf_program__pin(struct bpf_program *prog, const char *path)
1718 {
1719         int i, err;
1720
1721         err = check_path(path);
1722         if (err)
1723                 return err;
1724
1725         if (prog == NULL) {
1726                 pr_warning("invalid program pointer\n");
1727                 return -EINVAL;
1728         }
1729
1730         if (prog->instances.nr <= 0) {
1731                 pr_warning("no instances of prog %s to pin\n",
1732                            prog->section_name);
1733                 return -EINVAL;
1734         }
1735
1736         err = make_dir(path);
1737         if (err)
1738                 return err;
1739
1740         for (i = 0; i < prog->instances.nr; i++) {
1741                 char buf[PATH_MAX];
1742                 int len;
1743
1744                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1745                 if (len < 0)
1746                         return -EINVAL;
1747                 else if (len >= PATH_MAX)
1748                         return -ENAMETOOLONG;
1749
1750                 err = bpf_program__pin_instance(prog, buf, i);
1751                 if (err)
1752                         return err;
1753         }
1754
1755         return 0;
1756 }
1757
1758 int bpf_map__pin(struct bpf_map *map, const char *path)
1759 {
1760         char *cp, errmsg[STRERR_BUFSIZE];
1761         int err;
1762
1763         err = check_path(path);
1764         if (err)
1765                 return err;
1766
1767         if (map == NULL) {
1768                 pr_warning("invalid map pointer\n");
1769                 return -EINVAL;
1770         }
1771
1772         if (bpf_obj_pin(map->fd, path)) {
1773                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1774                 pr_warning("failed to pin map: %s\n", cp);
1775                 return -errno;
1776         }
1777
1778         pr_debug("pinned map '%s'\n", path);
1779         return 0;
1780 }
1781
1782 int bpf_object__pin(struct bpf_object *obj, const char *path)
1783 {
1784         struct bpf_program *prog;
1785         struct bpf_map *map;
1786         int err;
1787
1788         if (!obj)
1789                 return -ENOENT;
1790
1791         if (!obj->loaded) {
1792                 pr_warning("object not yet loaded; load it first\n");
1793                 return -ENOENT;
1794         }
1795
1796         err = make_dir(path);
1797         if (err)
1798                 return err;
1799
1800         bpf_map__for_each(map, obj) {
1801                 char buf[PATH_MAX];
1802                 int len;
1803
1804                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1805                                bpf_map__name(map));
1806                 if (len < 0)
1807                         return -EINVAL;
1808                 else if (len >= PATH_MAX)
1809                         return -ENAMETOOLONG;
1810
1811                 err = bpf_map__pin(map, buf);
1812                 if (err)
1813                         return err;
1814         }
1815
1816         bpf_object__for_each_program(prog, obj) {
1817                 char buf[PATH_MAX];
1818                 int len;
1819
1820                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1821                                prog->section_name);
1822                 if (len < 0)
1823                         return -EINVAL;
1824                 else if (len >= PATH_MAX)
1825                         return -ENAMETOOLONG;
1826
1827                 err = bpf_program__pin(prog, buf);
1828                 if (err)
1829                         return err;
1830         }
1831
1832         return 0;
1833 }
1834
1835 void bpf_object__close(struct bpf_object *obj)
1836 {
1837         size_t i;
1838
1839         if (!obj)
1840                 return;
1841
1842         if (obj->clear_priv)
1843                 obj->clear_priv(obj, obj->priv);
1844
1845         bpf_object__elf_finish(obj);
1846         bpf_object__unload(obj);
1847         btf__free(obj->btf);
1848
1849         for (i = 0; i < obj->nr_maps; i++) {
1850                 zfree(&obj->maps[i].name);
1851                 if (obj->maps[i].clear_priv)
1852                         obj->maps[i].clear_priv(&obj->maps[i],
1853                                                 obj->maps[i].priv);
1854                 obj->maps[i].priv = NULL;
1855                 obj->maps[i].clear_priv = NULL;
1856         }
1857         zfree(&obj->maps);
1858         obj->nr_maps = 0;
1859
1860         if (obj->programs && obj->nr_programs) {
1861                 for (i = 0; i < obj->nr_programs; i++)
1862                         bpf_program__exit(&obj->programs[i]);
1863         }
1864         zfree(&obj->programs);
1865
1866         list_del(&obj->list);
1867         free(obj);
1868 }
1869
1870 struct bpf_object *
1871 bpf_object__next(struct bpf_object *prev)
1872 {
1873         struct bpf_object *next;
1874
1875         if (!prev)
1876                 next = list_first_entry(&bpf_objects_list,
1877                                         struct bpf_object,
1878                                         list);
1879         else
1880                 next = list_next_entry(prev, list);
1881
1882         /* Empty list is noticed here so don't need checking on entry. */
1883         if (&next->list == &bpf_objects_list)
1884                 return NULL;
1885
1886         return next;
1887 }
1888
1889 const char *bpf_object__name(struct bpf_object *obj)
1890 {
1891         return obj ? obj->path : ERR_PTR(-EINVAL);
1892 }
1893
1894 unsigned int bpf_object__kversion(struct bpf_object *obj)
1895 {
1896         return obj ? obj->kern_version : 0;
1897 }
1898
1899 int bpf_object__btf_fd(const struct bpf_object *obj)
1900 {
1901         return obj->btf ? btf__fd(obj->btf) : -1;
1902 }
1903
1904 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1905                          bpf_object_clear_priv_t clear_priv)
1906 {
1907         if (obj->priv && obj->clear_priv)
1908                 obj->clear_priv(obj, obj->priv);
1909
1910         obj->priv = priv;
1911         obj->clear_priv = clear_priv;
1912         return 0;
1913 }
1914
1915 void *bpf_object__priv(struct bpf_object *obj)
1916 {
1917         return obj ? obj->priv : ERR_PTR(-EINVAL);
1918 }
1919
1920 static struct bpf_program *
1921 __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1922 {
1923         size_t idx;
1924
1925         if (!obj->programs)
1926                 return NULL;
1927         /* First handler */
1928         if (prev == NULL)
1929                 return &obj->programs[0];
1930
1931         if (prev->obj != obj) {
1932                 pr_warning("error: program handler doesn't match object\n");
1933                 return NULL;
1934         }
1935
1936         idx = (prev - obj->programs) + 1;
1937         if (idx >= obj->nr_programs)
1938                 return NULL;
1939         return &obj->programs[idx];
1940 }
1941
1942 struct bpf_program *
1943 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1944 {
1945         struct bpf_program *prog = prev;
1946
1947         do {
1948                 prog = __bpf_program__next(prog, obj);
1949         } while (prog && bpf_program__is_function_storage(prog, obj));
1950
1951         return prog;
1952 }
1953
1954 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1955                           bpf_program_clear_priv_t clear_priv)
1956 {
1957         if (prog->priv && prog->clear_priv)
1958                 prog->clear_priv(prog, prog->priv);
1959
1960         prog->priv = priv;
1961         prog->clear_priv = clear_priv;
1962         return 0;
1963 }
1964
1965 void *bpf_program__priv(struct bpf_program *prog)
1966 {
1967         return prog ? prog->priv : ERR_PTR(-EINVAL);
1968 }
1969
1970 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
1971 {
1972         prog->prog_ifindex = ifindex;
1973 }
1974
1975 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1976 {
1977         const char *title;
1978
1979         title = prog->section_name;
1980         if (needs_copy) {
1981                 title = strdup(title);
1982                 if (!title) {
1983                         pr_warning("failed to strdup program title\n");
1984                         return ERR_PTR(-ENOMEM);
1985                 }
1986         }
1987
1988         return title;
1989 }
1990
1991 int bpf_program__fd(struct bpf_program *prog)
1992 {
1993         return bpf_program__nth_fd(prog, 0);
1994 }
1995
1996 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1997                           bpf_program_prep_t prep)
1998 {
1999         int *instances_fds;
2000
2001         if (nr_instances <= 0 || !prep)
2002                 return -EINVAL;
2003
2004         if (prog->instances.nr > 0 || prog->instances.fds) {
2005                 pr_warning("Can't set pre-processor after loading\n");
2006                 return -EINVAL;
2007         }
2008
2009         instances_fds = malloc(sizeof(int) * nr_instances);
2010         if (!instances_fds) {
2011                 pr_warning("alloc memory failed for fds\n");
2012                 return -ENOMEM;
2013         }
2014
2015         /* fill all fd with -1 */
2016         memset(instances_fds, -1, sizeof(int) * nr_instances);
2017
2018         prog->instances.nr = nr_instances;
2019         prog->instances.fds = instances_fds;
2020         prog->preprocessor = prep;
2021         return 0;
2022 }
2023
2024 int bpf_program__nth_fd(struct bpf_program *prog, int n)
2025 {
2026         int fd;
2027
2028         if (!prog)
2029                 return -EINVAL;
2030
2031         if (n >= prog->instances.nr || n < 0) {
2032                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
2033                            n, prog->section_name, prog->instances.nr);
2034                 return -EINVAL;
2035         }
2036
2037         fd = prog->instances.fds[n];
2038         if (fd < 0) {
2039                 pr_warning("%dth instance of program '%s' is invalid\n",
2040                            n, prog->section_name);
2041                 return -ENOENT;
2042         }
2043
2044         return fd;
2045 }
2046
2047 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
2048 {
2049         prog->type = type;
2050 }
2051
2052 static bool bpf_program__is_type(struct bpf_program *prog,
2053                                  enum bpf_prog_type type)
2054 {
2055         return prog ? (prog->type == type) : false;
2056 }
2057
2058 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
2059 int bpf_program__set_##NAME(struct bpf_program *prog)   \
2060 {                                                       \
2061         if (!prog)                                      \
2062                 return -EINVAL;                         \
2063         bpf_program__set_type(prog, TYPE);              \
2064         return 0;                                       \
2065 }                                                       \
2066                                                         \
2067 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
2068 {                                                       \
2069         return bpf_program__is_type(prog, TYPE);        \
2070 }                                                       \
2071
2072 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
2073 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
2074 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
2075 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
2076 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
2077 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
2078 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
2079 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
2080
2081 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
2082                                            enum bpf_attach_type type)
2083 {
2084         prog->expected_attach_type = type;
2085 }
2086
2087 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \
2088         { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }
2089
2090 /* Programs that can NOT be attached. */
2091 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)
2092
2093 /* Programs that can be attached. */
2094 #define BPF_APROG_SEC(string, ptype, atype) \
2095         BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)
2096
2097 /* Programs that must specify expected attach type at load time. */
2098 #define BPF_EAPROG_SEC(string, ptype, eatype) \
2099         BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)
2100
2101 /* Programs that can be attached but attach type can't be identified by section
2102  * name. Kept for backward compatibility.
2103  */
2104 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
2105
2106 static const struct {
2107         const char *sec;
2108         size_t len;
2109         enum bpf_prog_type prog_type;
2110         enum bpf_attach_type expected_attach_type;
2111         int is_attachable;
2112         enum bpf_attach_type attach_type;
2113 } section_names[] = {
2114         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
2115         BPF_PROG_SEC("kprobe/",                 BPF_PROG_TYPE_KPROBE),
2116         BPF_PROG_SEC("kretprobe/",              BPF_PROG_TYPE_KPROBE),
2117         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
2118         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
2119         BPF_PROG_SEC("tracepoint/",             BPF_PROG_TYPE_TRACEPOINT),
2120         BPF_PROG_SEC("raw_tracepoint/",         BPF_PROG_TYPE_RAW_TRACEPOINT),
2121         BPF_PROG_SEC("xdp",                     BPF_PROG_TYPE_XDP),
2122         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
2123         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
2124         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
2125         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
2126         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
2127         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
2128                                                 BPF_CGROUP_INET_INGRESS),
2129         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
2130                                                 BPF_CGROUP_INET_EGRESS),
2131         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
2132         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
2133                                                 BPF_CGROUP_INET_SOCK_CREATE),
2134         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
2135                                                 BPF_CGROUP_INET4_POST_BIND),
2136         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
2137                                                 BPF_CGROUP_INET6_POST_BIND),
2138         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
2139                                                 BPF_CGROUP_DEVICE),
2140         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
2141                                                 BPF_CGROUP_SOCK_OPS),
2142         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
2143                                                 BPF_SK_SKB_STREAM_PARSER),
2144         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
2145                                                 BPF_SK_SKB_STREAM_VERDICT),
2146         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
2147         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
2148                                                 BPF_SK_MSG_VERDICT),
2149         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
2150                                                 BPF_LIRC_MODE2),
2151         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
2152                                                 BPF_FLOW_DISSECTOR),
2153         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2154                                                 BPF_CGROUP_INET4_BIND),
2155         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2156                                                 BPF_CGROUP_INET6_BIND),
2157         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2158                                                 BPF_CGROUP_INET4_CONNECT),
2159         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2160                                                 BPF_CGROUP_INET6_CONNECT),
2161         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2162                                                 BPF_CGROUP_UDP4_SENDMSG),
2163         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2164                                                 BPF_CGROUP_UDP6_SENDMSG),
2165 };
2166
2167 #undef BPF_PROG_SEC_IMPL
2168 #undef BPF_PROG_SEC
2169 #undef BPF_APROG_SEC
2170 #undef BPF_EAPROG_SEC
2171 #undef BPF_APROG_COMPAT
2172
2173 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
2174                              enum bpf_attach_type *expected_attach_type)
2175 {
2176         int i;
2177
2178         if (!name)
2179                 return -EINVAL;
2180
2181         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2182                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2183                         continue;
2184                 *prog_type = section_names[i].prog_type;
2185                 *expected_attach_type = section_names[i].expected_attach_type;
2186                 return 0;
2187         }
2188         return -EINVAL;
2189 }
2190
2191 int libbpf_attach_type_by_name(const char *name,
2192                                enum bpf_attach_type *attach_type)
2193 {
2194         int i;
2195
2196         if (!name)
2197                 return -EINVAL;
2198
2199         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2200                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2201                         continue;
2202                 if (!section_names[i].is_attachable)
2203                         return -EINVAL;
2204                 *attach_type = section_names[i].attach_type;
2205                 return 0;
2206         }
2207         return -EINVAL;
2208 }
2209
2210 static int
2211 bpf_program__identify_section(struct bpf_program *prog,
2212                               enum bpf_prog_type *prog_type,
2213                               enum bpf_attach_type *expected_attach_type)
2214 {
2215         return libbpf_prog_type_by_name(prog->section_name, prog_type,
2216                                         expected_attach_type);
2217 }
2218
2219 int bpf_map__fd(struct bpf_map *map)
2220 {
2221         return map ? map->fd : -EINVAL;
2222 }
2223
2224 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
2225 {
2226         return map ? &map->def : ERR_PTR(-EINVAL);
2227 }
2228
2229 const char *bpf_map__name(struct bpf_map *map)
2230 {
2231         return map ? map->name : NULL;
2232 }
2233
2234 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
2235 {
2236         return map ? map->btf_key_type_id : 0;
2237 }
2238
2239 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
2240 {
2241         return map ? map->btf_value_type_id : 0;
2242 }
2243
2244 int bpf_map__set_priv(struct bpf_map *map, void *priv,
2245                      bpf_map_clear_priv_t clear_priv)
2246 {
2247         if (!map)
2248                 return -EINVAL;
2249
2250         if (map->priv) {
2251                 if (map->clear_priv)
2252                         map->clear_priv(map, map->priv);
2253         }
2254
2255         map->priv = priv;
2256         map->clear_priv = clear_priv;
2257         return 0;
2258 }
2259
2260 void *bpf_map__priv(struct bpf_map *map)
2261 {
2262         return map ? map->priv : ERR_PTR(-EINVAL);
2263 }
2264
2265 bool bpf_map__is_offload_neutral(struct bpf_map *map)
2266 {
2267         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2268 }
2269
2270 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2271 {
2272         map->map_ifindex = ifindex;
2273 }
2274
2275 struct bpf_map *
2276 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
2277 {
2278         size_t idx;
2279         struct bpf_map *s, *e;
2280
2281         if (!obj || !obj->maps)
2282                 return NULL;
2283
2284         s = obj->maps;
2285         e = obj->maps + obj->nr_maps;
2286
2287         if (prev == NULL)
2288                 return s;
2289
2290         if ((prev < s) || (prev >= e)) {
2291                 pr_warning("error in %s: map handler doesn't belong to object\n",
2292                            __func__);
2293                 return NULL;
2294         }
2295
2296         idx = (prev - obj->maps) + 1;
2297         if (idx >= obj->nr_maps)
2298                 return NULL;
2299         return &obj->maps[idx];
2300 }
2301
2302 struct bpf_map *
2303 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
2304 {
2305         struct bpf_map *pos;
2306
2307         bpf_map__for_each(pos, obj) {
2308                 if (pos->name && !strcmp(pos->name, name))
2309                         return pos;
2310         }
2311         return NULL;
2312 }
2313
2314 struct bpf_map *
2315 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
2316 {
2317         int i;
2318
2319         for (i = 0; i < obj->nr_maps; i++) {
2320                 if (obj->maps[i].offset == offset)
2321                         return &obj->maps[i];
2322         }
2323         return ERR_PTR(-ENOENT);
2324 }
2325
2326 long libbpf_get_error(const void *ptr)
2327 {
2328         if (IS_ERR(ptr))
2329                 return PTR_ERR(ptr);
2330         return 0;
2331 }
2332
2333 int bpf_prog_load(const char *file, enum bpf_prog_type type,
2334                   struct bpf_object **pobj, int *prog_fd)
2335 {
2336         struct bpf_prog_load_attr attr;
2337
2338         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2339         attr.file = file;
2340         attr.prog_type = type;
2341         attr.expected_attach_type = 0;
2342
2343         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2344 }
2345
2346 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2347                         struct bpf_object **pobj, int *prog_fd)
2348 {
2349         struct bpf_object_open_attr open_attr = {
2350                 .file           = attr->file,
2351                 .prog_type      = attr->prog_type,
2352         };
2353         struct bpf_program *prog, *first_prog = NULL;
2354         enum bpf_attach_type expected_attach_type;
2355         enum bpf_prog_type prog_type;
2356         struct bpf_object *obj;
2357         struct bpf_map *map;
2358         int err;
2359
2360         if (!attr)
2361                 return -EINVAL;
2362         if (!attr->file)
2363                 return -EINVAL;
2364
2365         obj = bpf_object__open_xattr(&open_attr);
2366         if (IS_ERR_OR_NULL(obj))
2367                 return -ENOENT;
2368
2369         bpf_object__for_each_program(prog, obj) {
2370                 /*
2371                  * If type is not specified, try to guess it based on
2372                  * section name.
2373                  */
2374                 prog_type = attr->prog_type;
2375                 prog->prog_ifindex = attr->ifindex;
2376                 expected_attach_type = attr->expected_attach_type;
2377                 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2378                         err = bpf_program__identify_section(prog, &prog_type,
2379                                                             &expected_attach_type);
2380                         if (err < 0) {
2381                                 pr_warning("failed to guess program type based on section name %s\n",
2382                                            prog->section_name);
2383                                 bpf_object__close(obj);
2384                                 return -EINVAL;
2385                         }
2386                 }
2387
2388                 bpf_program__set_type(prog, prog_type);
2389                 bpf_program__set_expected_attach_type(prog,
2390                                                       expected_attach_type);
2391
2392                 if (!first_prog)
2393                         first_prog = prog;
2394         }
2395
2396         bpf_map__for_each(map, obj) {
2397                 if (!bpf_map__is_offload_neutral(map))
2398                         map->map_ifindex = attr->ifindex;
2399         }
2400
2401         if (!first_prog) {
2402                 pr_warning("object file doesn't contain bpf program\n");
2403                 bpf_object__close(obj);
2404                 return -ENOENT;
2405         }
2406
2407         err = bpf_object__load(obj);
2408         if (err) {
2409                 bpf_object__close(obj);
2410                 return -EINVAL;
2411         }
2412
2413         *pobj = obj;
2414         *prog_fd = bpf_program__fd(first_prog);
2415         return 0;
2416 }
2417
2418 enum bpf_perf_event_ret
2419 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
2420                            void **copy_mem, size_t *copy_size,
2421                            bpf_perf_event_print_t fn, void *private_data)
2422 {
2423         struct perf_event_mmap_page *header = mmap_mem;
2424         __u64 data_head = ring_buffer_read_head(header);
2425         __u64 data_tail = header->data_tail;
2426         void *base = ((__u8 *)header) + page_size;
2427         int ret = LIBBPF_PERF_EVENT_CONT;
2428         struct perf_event_header *ehdr;
2429         size_t ehdr_size;
2430
2431         while (data_head != data_tail) {
2432                 ehdr = base + (data_tail & (mmap_size - 1));
2433                 ehdr_size = ehdr->size;
2434
2435                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
2436                         void *copy_start = ehdr;
2437                         size_t len_first = base + mmap_size - copy_start;
2438                         size_t len_secnd = ehdr_size - len_first;
2439
2440                         if (*copy_size < ehdr_size) {
2441                                 free(*copy_mem);
2442                                 *copy_mem = malloc(ehdr_size);
2443                                 if (!*copy_mem) {
2444                                         *copy_size = 0;
2445                                         ret = LIBBPF_PERF_EVENT_ERROR;
2446                                         break;
2447                                 }
2448                                 *copy_size = ehdr_size;
2449                         }
2450
2451                         memcpy(*copy_mem, copy_start, len_first);
2452                         memcpy(*copy_mem + len_first, base, len_secnd);
2453                         ehdr = *copy_mem;
2454                 }
2455
2456                 ret = fn(ehdr, private_data);
2457                 data_tail += ehdr_size;
2458                 if (ret != LIBBPF_PERF_EVENT_CONT)
2459                         break;
2460         }
2461
2462         ring_buffer_write_tail(header, data_tail);
2463         return ret;
2464 }