Merge tag 'linux-watchdog-5.1-rc1' of git://www.linux-watchdog.org/linux-watchdog
[muen/linux.git] / kernel / trace / trace_events_hist.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/mutex.h>
11 #include <linux/slab.h>
12 #include <linux/stacktrace.h>
13 #include <linux/rculist.h>
14 #include <linux/tracefs.h>
15
16 #include "tracing_map.h"
17 #include "trace.h"
18 #include "trace_dynevent.h"
19
20 #define SYNTH_SYSTEM            "synthetic"
21 #define SYNTH_FIELDS_MAX        16
22
23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
24
25 struct hist_field;
26
27 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
28                                 struct tracing_map_elt *elt,
29                                 struct ring_buffer_event *rbe,
30                                 void *event);
31
32 #define HIST_FIELD_OPERANDS_MAX 2
33 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
34 #define HIST_ACTIONS_MAX        8
35
36 enum field_op_id {
37         FIELD_OP_NONE,
38         FIELD_OP_PLUS,
39         FIELD_OP_MINUS,
40         FIELD_OP_UNARY_MINUS,
41 };
42
43 /*
44  * A hist_var (histogram variable) contains variable information for
45  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
46  * flag set.  A hist_var has a variable name e.g. ts0, and is
47  * associated with a given histogram trigger, as specified by
48  * hist_data.  The hist_var idx is the unique index assigned to the
49  * variable by the hist trigger's tracing_map.  The idx is what is
50  * used to set a variable's value and, by a variable reference, to
51  * retrieve it.
52  */
53 struct hist_var {
54         char                            *name;
55         struct hist_trigger_data        *hist_data;
56         unsigned int                    idx;
57 };
58
59 struct hist_field {
60         struct ftrace_event_field       *field;
61         unsigned long                   flags;
62         hist_field_fn_t                 fn;
63         unsigned int                    size;
64         unsigned int                    offset;
65         unsigned int                    is_signed;
66         const char                      *type;
67         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
68         struct hist_trigger_data        *hist_data;
69
70         /*
71          * Variable fields contain variable-specific info in var.
72          */
73         struct hist_var                 var;
74         enum field_op_id                operator;
75         char                            *system;
76         char                            *event_name;
77
78         /*
79          * The name field is used for EXPR and VAR_REF fields.  VAR
80          * fields contain the variable name in var.name.
81          */
82         char                            *name;
83
84         /*
85          * When a histogram trigger is hit, if it has any references
86          * to variables, the values of those variables are collected
87          * into a var_ref_vals array by resolve_var_refs().  The
88          * current value of each variable is read from the tracing_map
89          * using the hist field's hist_var.idx and entered into the
90          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
91          */
92         unsigned int                    var_ref_idx;
93         bool                            read_once;
94 };
95
96 static u64 hist_field_none(struct hist_field *field,
97                            struct tracing_map_elt *elt,
98                            struct ring_buffer_event *rbe,
99                            void *event)
100 {
101         return 0;
102 }
103
104 static u64 hist_field_counter(struct hist_field *field,
105                               struct tracing_map_elt *elt,
106                               struct ring_buffer_event *rbe,
107                               void *event)
108 {
109         return 1;
110 }
111
112 static u64 hist_field_string(struct hist_field *hist_field,
113                              struct tracing_map_elt *elt,
114                              struct ring_buffer_event *rbe,
115                              void *event)
116 {
117         char *addr = (char *)(event + hist_field->field->offset);
118
119         return (u64)(unsigned long)addr;
120 }
121
122 static u64 hist_field_dynstring(struct hist_field *hist_field,
123                                 struct tracing_map_elt *elt,
124                                 struct ring_buffer_event *rbe,
125                                 void *event)
126 {
127         u32 str_item = *(u32 *)(event + hist_field->field->offset);
128         int str_loc = str_item & 0xffff;
129         char *addr = (char *)(event + str_loc);
130
131         return (u64)(unsigned long)addr;
132 }
133
134 static u64 hist_field_pstring(struct hist_field *hist_field,
135                               struct tracing_map_elt *elt,
136                               struct ring_buffer_event *rbe,
137                               void *event)
138 {
139         char **addr = (char **)(event + hist_field->field->offset);
140
141         return (u64)(unsigned long)*addr;
142 }
143
144 static u64 hist_field_log2(struct hist_field *hist_field,
145                            struct tracing_map_elt *elt,
146                            struct ring_buffer_event *rbe,
147                            void *event)
148 {
149         struct hist_field *operand = hist_field->operands[0];
150
151         u64 val = operand->fn(operand, elt, rbe, event);
152
153         return (u64) ilog2(roundup_pow_of_two(val));
154 }
155
156 static u64 hist_field_plus(struct hist_field *hist_field,
157                            struct tracing_map_elt *elt,
158                            struct ring_buffer_event *rbe,
159                            void *event)
160 {
161         struct hist_field *operand1 = hist_field->operands[0];
162         struct hist_field *operand2 = hist_field->operands[1];
163
164         u64 val1 = operand1->fn(operand1, elt, rbe, event);
165         u64 val2 = operand2->fn(operand2, elt, rbe, event);
166
167         return val1 + val2;
168 }
169
170 static u64 hist_field_minus(struct hist_field *hist_field,
171                             struct tracing_map_elt *elt,
172                             struct ring_buffer_event *rbe,
173                             void *event)
174 {
175         struct hist_field *operand1 = hist_field->operands[0];
176         struct hist_field *operand2 = hist_field->operands[1];
177
178         u64 val1 = operand1->fn(operand1, elt, rbe, event);
179         u64 val2 = operand2->fn(operand2, elt, rbe, event);
180
181         return val1 - val2;
182 }
183
184 static u64 hist_field_unary_minus(struct hist_field *hist_field,
185                                   struct tracing_map_elt *elt,
186                                   struct ring_buffer_event *rbe,
187                                   void *event)
188 {
189         struct hist_field *operand = hist_field->operands[0];
190
191         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
192         u64 val = (u64)-sval;
193
194         return val;
195 }
196
197 #define DEFINE_HIST_FIELD_FN(type)                                      \
198         static u64 hist_field_##type(struct hist_field *hist_field,     \
199                                      struct tracing_map_elt *elt,       \
200                                      struct ring_buffer_event *rbe,     \
201                                      void *event)                       \
202 {                                                                       \
203         type *addr = (type *)(event + hist_field->field->offset);       \
204                                                                         \
205         return (u64)(unsigned long)*addr;                               \
206 }
207
208 DEFINE_HIST_FIELD_FN(s64);
209 DEFINE_HIST_FIELD_FN(u64);
210 DEFINE_HIST_FIELD_FN(s32);
211 DEFINE_HIST_FIELD_FN(u32);
212 DEFINE_HIST_FIELD_FN(s16);
213 DEFINE_HIST_FIELD_FN(u16);
214 DEFINE_HIST_FIELD_FN(s8);
215 DEFINE_HIST_FIELD_FN(u8);
216
217 #define for_each_hist_field(i, hist_data)       \
218         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
219
220 #define for_each_hist_val_field(i, hist_data)   \
221         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
222
223 #define for_each_hist_key_field(i, hist_data)   \
224         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
225
226 #define HIST_STACKTRACE_DEPTH   16
227 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
228 #define HIST_STACKTRACE_SKIP    5
229
230 #define HITCOUNT_IDX            0
231 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
232
233 enum hist_field_flags {
234         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
235         HIST_FIELD_FL_KEY               = 1 << 1,
236         HIST_FIELD_FL_STRING            = 1 << 2,
237         HIST_FIELD_FL_HEX               = 1 << 3,
238         HIST_FIELD_FL_SYM               = 1 << 4,
239         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
240         HIST_FIELD_FL_EXECNAME          = 1 << 6,
241         HIST_FIELD_FL_SYSCALL           = 1 << 7,
242         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
243         HIST_FIELD_FL_LOG2              = 1 << 9,
244         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
245         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
246         HIST_FIELD_FL_VAR               = 1 << 12,
247         HIST_FIELD_FL_EXPR              = 1 << 13,
248         HIST_FIELD_FL_VAR_REF           = 1 << 14,
249         HIST_FIELD_FL_CPU               = 1 << 15,
250         HIST_FIELD_FL_ALIAS             = 1 << 16,
251 };
252
253 struct var_defs {
254         unsigned int    n_vars;
255         char            *name[TRACING_MAP_VARS_MAX];
256         char            *expr[TRACING_MAP_VARS_MAX];
257 };
258
259 struct hist_trigger_attrs {
260         char            *keys_str;
261         char            *vals_str;
262         char            *sort_key_str;
263         char            *name;
264         char            *clock;
265         bool            pause;
266         bool            cont;
267         bool            clear;
268         bool            ts_in_usecs;
269         unsigned int    map_bits;
270
271         char            *assignment_str[TRACING_MAP_VARS_MAX];
272         unsigned int    n_assignments;
273
274         char            *action_str[HIST_ACTIONS_MAX];
275         unsigned int    n_actions;
276
277         struct var_defs var_defs;
278 };
279
280 struct field_var {
281         struct hist_field       *var;
282         struct hist_field       *val;
283 };
284
285 struct field_var_hist {
286         struct hist_trigger_data        *hist_data;
287         char                            *cmd;
288 };
289
290 struct hist_trigger_data {
291         struct hist_field               *fields[HIST_FIELDS_MAX];
292         unsigned int                    n_vals;
293         unsigned int                    n_keys;
294         unsigned int                    n_fields;
295         unsigned int                    n_vars;
296         unsigned int                    key_size;
297         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
298         unsigned int                    n_sort_keys;
299         struct trace_event_file         *event_file;
300         struct hist_trigger_attrs       *attrs;
301         struct tracing_map              *map;
302         bool                            enable_timestamps;
303         bool                            remove;
304         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
305         unsigned int                    n_var_refs;
306
307         struct action_data              *actions[HIST_ACTIONS_MAX];
308         unsigned int                    n_actions;
309
310         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
311         unsigned int                    n_field_vars;
312         unsigned int                    n_field_var_str;
313         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
314         unsigned int                    n_field_var_hists;
315
316         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
317         unsigned int                    n_max_vars;
318         unsigned int                    n_max_var_str;
319 };
320
321 static int synth_event_create(int argc, const char **argv);
322 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
323 static int synth_event_release(struct dyn_event *ev);
324 static bool synth_event_is_busy(struct dyn_event *ev);
325 static bool synth_event_match(const char *system, const char *event,
326                               struct dyn_event *ev);
327
328 static struct dyn_event_operations synth_event_ops = {
329         .create = synth_event_create,
330         .show = synth_event_show,
331         .is_busy = synth_event_is_busy,
332         .free = synth_event_release,
333         .match = synth_event_match,
334 };
335
336 struct synth_field {
337         char *type;
338         char *name;
339         size_t size;
340         bool is_signed;
341         bool is_string;
342 };
343
344 struct synth_event {
345         struct dyn_event                        devent;
346         int                                     ref;
347         char                                    *name;
348         struct synth_field                      **fields;
349         unsigned int                            n_fields;
350         unsigned int                            n_u64;
351         struct trace_event_class                class;
352         struct trace_event_call                 call;
353         struct tracepoint                       *tp;
354 };
355
356 static bool is_synth_event(struct dyn_event *ev)
357 {
358         return ev->ops == &synth_event_ops;
359 }
360
361 static struct synth_event *to_synth_event(struct dyn_event *ev)
362 {
363         return container_of(ev, struct synth_event, devent);
364 }
365
366 static bool synth_event_is_busy(struct dyn_event *ev)
367 {
368         struct synth_event *event = to_synth_event(ev);
369
370         return event->ref != 0;
371 }
372
373 static bool synth_event_match(const char *system, const char *event,
374                               struct dyn_event *ev)
375 {
376         struct synth_event *sev = to_synth_event(ev);
377
378         return strcmp(sev->name, event) == 0 &&
379                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
380 }
381
382 struct action_data;
383
384 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
385                              struct tracing_map_elt *elt, void *rec,
386                              struct ring_buffer_event *rbe,
387                              struct action_data *data, u64 *var_ref_vals);
388
389 struct action_data {
390         action_fn_t             fn;
391         unsigned int            n_params;
392         char                    *params[SYNTH_FIELDS_MAX];
393
394         union {
395                 struct {
396                         /*
397                          * When a histogram trigger is hit, the values of any
398                          * references to variables, including variables being passed
399                          * as parameters to synthetic events, are collected into a
400                          * var_ref_vals array.  This var_ref_idx is the index of the
401                          * first param in the array to be passed to the synthetic
402                          * event invocation.
403                          */
404                         unsigned int            var_ref_idx;
405                         char                    *match_event;
406                         char                    *match_event_system;
407                         char                    *synth_event_name;
408                         struct synth_event      *synth_event;
409                 } onmatch;
410
411                 struct {
412                         char                    *var_str;
413                         char                    *fn_name;
414                         unsigned int            max_var_ref_idx;
415                         struct hist_field       *max_var;
416                         struct hist_field       *var;
417                 } onmax;
418         };
419 };
420
421
422 static char last_hist_cmd[MAX_FILTER_STR_VAL];
423 static char hist_err_str[MAX_FILTER_STR_VAL];
424
425 static void last_cmd_set(char *str)
426 {
427         if (!str)
428                 return;
429
430         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
431 }
432
433 static void hist_err(char *str, char *var)
434 {
435         int maxlen = MAX_FILTER_STR_VAL - 1;
436
437         if (!str)
438                 return;
439
440         if (strlen(hist_err_str))
441                 return;
442
443         if (!var)
444                 var = "";
445
446         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
447                 return;
448
449         strcat(hist_err_str, str);
450         strcat(hist_err_str, var);
451 }
452
453 static void hist_err_event(char *str, char *system, char *event, char *var)
454 {
455         char err[MAX_FILTER_STR_VAL];
456
457         if (system && var)
458                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
459         else if (system)
460                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
461         else
462                 strscpy(err, var, MAX_FILTER_STR_VAL);
463
464         hist_err(str, err);
465 }
466
467 static void hist_err_clear(void)
468 {
469         hist_err_str[0] = '\0';
470 }
471
472 static bool have_hist_err(void)
473 {
474         if (strlen(hist_err_str))
475                 return true;
476
477         return false;
478 }
479
480 struct synth_trace_event {
481         struct trace_entry      ent;
482         u64                     fields[];
483 };
484
485 static int synth_event_define_fields(struct trace_event_call *call)
486 {
487         struct synth_trace_event trace;
488         int offset = offsetof(typeof(trace), fields);
489         struct synth_event *event = call->data;
490         unsigned int i, size, n_u64;
491         char *name, *type;
492         bool is_signed;
493         int ret = 0;
494
495         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
496                 size = event->fields[i]->size;
497                 is_signed = event->fields[i]->is_signed;
498                 type = event->fields[i]->type;
499                 name = event->fields[i]->name;
500                 ret = trace_define_field(call, type, name, offset, size,
501                                          is_signed, FILTER_OTHER);
502                 if (ret)
503                         break;
504
505                 if (event->fields[i]->is_string) {
506                         offset += STR_VAR_LEN_MAX;
507                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
508                 } else {
509                         offset += sizeof(u64);
510                         n_u64++;
511                 }
512         }
513
514         event->n_u64 = n_u64;
515
516         return ret;
517 }
518
519 static bool synth_field_signed(char *type)
520 {
521         if (str_has_prefix(type, "u"))
522                 return false;
523
524         return true;
525 }
526
527 static int synth_field_is_string(char *type)
528 {
529         if (strstr(type, "char[") != NULL)
530                 return true;
531
532         return false;
533 }
534
535 static int synth_field_string_size(char *type)
536 {
537         char buf[4], *end, *start;
538         unsigned int len;
539         int size, err;
540
541         start = strstr(type, "char[");
542         if (start == NULL)
543                 return -EINVAL;
544         start += sizeof("char[") - 1;
545
546         end = strchr(type, ']');
547         if (!end || end < start)
548                 return -EINVAL;
549
550         len = end - start;
551         if (len > 3)
552                 return -EINVAL;
553
554         strncpy(buf, start, len);
555         buf[len] = '\0';
556
557         err = kstrtouint(buf, 0, &size);
558         if (err)
559                 return err;
560
561         if (size > STR_VAR_LEN_MAX)
562                 return -EINVAL;
563
564         return size;
565 }
566
567 static int synth_field_size(char *type)
568 {
569         int size = 0;
570
571         if (strcmp(type, "s64") == 0)
572                 size = sizeof(s64);
573         else if (strcmp(type, "u64") == 0)
574                 size = sizeof(u64);
575         else if (strcmp(type, "s32") == 0)
576                 size = sizeof(s32);
577         else if (strcmp(type, "u32") == 0)
578                 size = sizeof(u32);
579         else if (strcmp(type, "s16") == 0)
580                 size = sizeof(s16);
581         else if (strcmp(type, "u16") == 0)
582                 size = sizeof(u16);
583         else if (strcmp(type, "s8") == 0)
584                 size = sizeof(s8);
585         else if (strcmp(type, "u8") == 0)
586                 size = sizeof(u8);
587         else if (strcmp(type, "char") == 0)
588                 size = sizeof(char);
589         else if (strcmp(type, "unsigned char") == 0)
590                 size = sizeof(unsigned char);
591         else if (strcmp(type, "int") == 0)
592                 size = sizeof(int);
593         else if (strcmp(type, "unsigned int") == 0)
594                 size = sizeof(unsigned int);
595         else if (strcmp(type, "long") == 0)
596                 size = sizeof(long);
597         else if (strcmp(type, "unsigned long") == 0)
598                 size = sizeof(unsigned long);
599         else if (strcmp(type, "pid_t") == 0)
600                 size = sizeof(pid_t);
601         else if (synth_field_is_string(type))
602                 size = synth_field_string_size(type);
603
604         return size;
605 }
606
607 static const char *synth_field_fmt(char *type)
608 {
609         const char *fmt = "%llu";
610
611         if (strcmp(type, "s64") == 0)
612                 fmt = "%lld";
613         else if (strcmp(type, "u64") == 0)
614                 fmt = "%llu";
615         else if (strcmp(type, "s32") == 0)
616                 fmt = "%d";
617         else if (strcmp(type, "u32") == 0)
618                 fmt = "%u";
619         else if (strcmp(type, "s16") == 0)
620                 fmt = "%d";
621         else if (strcmp(type, "u16") == 0)
622                 fmt = "%u";
623         else if (strcmp(type, "s8") == 0)
624                 fmt = "%d";
625         else if (strcmp(type, "u8") == 0)
626                 fmt = "%u";
627         else if (strcmp(type, "char") == 0)
628                 fmt = "%d";
629         else if (strcmp(type, "unsigned char") == 0)
630                 fmt = "%u";
631         else if (strcmp(type, "int") == 0)
632                 fmt = "%d";
633         else if (strcmp(type, "unsigned int") == 0)
634                 fmt = "%u";
635         else if (strcmp(type, "long") == 0)
636                 fmt = "%ld";
637         else if (strcmp(type, "unsigned long") == 0)
638                 fmt = "%lu";
639         else if (strcmp(type, "pid_t") == 0)
640                 fmt = "%d";
641         else if (synth_field_is_string(type))
642                 fmt = "%s";
643
644         return fmt;
645 }
646
647 static enum print_line_t print_synth_event(struct trace_iterator *iter,
648                                            int flags,
649                                            struct trace_event *event)
650 {
651         struct trace_array *tr = iter->tr;
652         struct trace_seq *s = &iter->seq;
653         struct synth_trace_event *entry;
654         struct synth_event *se;
655         unsigned int i, n_u64;
656         char print_fmt[32];
657         const char *fmt;
658
659         entry = (struct synth_trace_event *)iter->ent;
660         se = container_of(event, struct synth_event, call.event);
661
662         trace_seq_printf(s, "%s: ", se->name);
663
664         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
665                 if (trace_seq_has_overflowed(s))
666                         goto end;
667
668                 fmt = synth_field_fmt(se->fields[i]->type);
669
670                 /* parameter types */
671                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
672                         trace_seq_printf(s, "%s ", fmt);
673
674                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
675
676                 /* parameter values */
677                 if (se->fields[i]->is_string) {
678                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
679                                          (char *)&entry->fields[n_u64],
680                                          i == se->n_fields - 1 ? "" : " ");
681                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
682                 } else {
683                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
684                                          entry->fields[n_u64],
685                                          i == se->n_fields - 1 ? "" : " ");
686                         n_u64++;
687                 }
688         }
689 end:
690         trace_seq_putc(s, '\n');
691
692         return trace_handle_return(s);
693 }
694
695 static struct trace_event_functions synth_event_funcs = {
696         .trace          = print_synth_event
697 };
698
699 static notrace void trace_event_raw_event_synth(void *__data,
700                                                 u64 *var_ref_vals,
701                                                 unsigned int var_ref_idx)
702 {
703         struct trace_event_file *trace_file = __data;
704         struct synth_trace_event *entry;
705         struct trace_event_buffer fbuffer;
706         struct ring_buffer *buffer;
707         struct synth_event *event;
708         unsigned int i, n_u64;
709         int fields_size = 0;
710
711         event = trace_file->event_call->data;
712
713         if (trace_trigger_soft_disabled(trace_file))
714                 return;
715
716         fields_size = event->n_u64 * sizeof(u64);
717
718         /*
719          * Avoid ring buffer recursion detection, as this event
720          * is being performed within another event.
721          */
722         buffer = trace_file->tr->trace_buffer.buffer;
723         ring_buffer_nest_start(buffer);
724
725         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
726                                            sizeof(*entry) + fields_size);
727         if (!entry)
728                 goto out;
729
730         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
731                 if (event->fields[i]->is_string) {
732                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
733                         char *str_field = (char *)&entry->fields[n_u64];
734
735                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
736                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
737                 } else {
738                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
739                         n_u64++;
740                 }
741         }
742
743         trace_event_buffer_commit(&fbuffer);
744 out:
745         ring_buffer_nest_end(buffer);
746 }
747
748 static void free_synth_event_print_fmt(struct trace_event_call *call)
749 {
750         if (call) {
751                 kfree(call->print_fmt);
752                 call->print_fmt = NULL;
753         }
754 }
755
756 static int __set_synth_event_print_fmt(struct synth_event *event,
757                                        char *buf, int len)
758 {
759         const char *fmt;
760         int pos = 0;
761         int i;
762
763         /* When len=0, we just calculate the needed length */
764 #define LEN_OR_ZERO (len ? len - pos : 0)
765
766         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
767         for (i = 0; i < event->n_fields; i++) {
768                 fmt = synth_field_fmt(event->fields[i]->type);
769                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
770                                 event->fields[i]->name, fmt,
771                                 i == event->n_fields - 1 ? "" : ", ");
772         }
773         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
774
775         for (i = 0; i < event->n_fields; i++) {
776                 pos += snprintf(buf + pos, LEN_OR_ZERO,
777                                 ", REC->%s", event->fields[i]->name);
778         }
779
780 #undef LEN_OR_ZERO
781
782         /* return the length of print_fmt */
783         return pos;
784 }
785
786 static int set_synth_event_print_fmt(struct trace_event_call *call)
787 {
788         struct synth_event *event = call->data;
789         char *print_fmt;
790         int len;
791
792         /* First: called with 0 length to calculate the needed length */
793         len = __set_synth_event_print_fmt(event, NULL, 0);
794
795         print_fmt = kmalloc(len + 1, GFP_KERNEL);
796         if (!print_fmt)
797                 return -ENOMEM;
798
799         /* Second: actually write the @print_fmt */
800         __set_synth_event_print_fmt(event, print_fmt, len + 1);
801         call->print_fmt = print_fmt;
802
803         return 0;
804 }
805
806 static void free_synth_field(struct synth_field *field)
807 {
808         kfree(field->type);
809         kfree(field->name);
810         kfree(field);
811 }
812
813 static struct synth_field *parse_synth_field(int argc, const char **argv,
814                                              int *consumed)
815 {
816         struct synth_field *field;
817         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
818         int len, ret = 0;
819
820         if (field_type[0] == ';')
821                 field_type++;
822
823         if (!strcmp(field_type, "unsigned")) {
824                 if (argc < 3)
825                         return ERR_PTR(-EINVAL);
826                 prefix = "unsigned ";
827                 field_type = argv[1];
828                 field_name = argv[2];
829                 *consumed = 3;
830         } else {
831                 field_name = argv[1];
832                 *consumed = 2;
833         }
834
835         field = kzalloc(sizeof(*field), GFP_KERNEL);
836         if (!field)
837                 return ERR_PTR(-ENOMEM);
838
839         len = strlen(field_name);
840         array = strchr(field_name, '[');
841         if (array)
842                 len -= strlen(array);
843         else if (field_name[len - 1] == ';')
844                 len--;
845
846         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
847         if (!field->name) {
848                 ret = -ENOMEM;
849                 goto free;
850         }
851
852         if (field_type[0] == ';')
853                 field_type++;
854         len = strlen(field_type) + 1;
855         if (array)
856                 len += strlen(array);
857         if (prefix)
858                 len += strlen(prefix);
859
860         field->type = kzalloc(len, GFP_KERNEL);
861         if (!field->type) {
862                 ret = -ENOMEM;
863                 goto free;
864         }
865         if (prefix)
866                 strcat(field->type, prefix);
867         strcat(field->type, field_type);
868         if (array) {
869                 strcat(field->type, array);
870                 if (field->type[len - 1] == ';')
871                         field->type[len - 1] = '\0';
872         }
873
874         field->size = synth_field_size(field->type);
875         if (!field->size) {
876                 ret = -EINVAL;
877                 goto free;
878         }
879
880         if (synth_field_is_string(field->type))
881                 field->is_string = true;
882
883         field->is_signed = synth_field_signed(field->type);
884
885  out:
886         return field;
887  free:
888         free_synth_field(field);
889         field = ERR_PTR(ret);
890         goto out;
891 }
892
893 static void free_synth_tracepoint(struct tracepoint *tp)
894 {
895         if (!tp)
896                 return;
897
898         kfree(tp->name);
899         kfree(tp);
900 }
901
902 static struct tracepoint *alloc_synth_tracepoint(char *name)
903 {
904         struct tracepoint *tp;
905
906         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
907         if (!tp)
908                 return ERR_PTR(-ENOMEM);
909
910         tp->name = kstrdup(name, GFP_KERNEL);
911         if (!tp->name) {
912                 kfree(tp);
913                 return ERR_PTR(-ENOMEM);
914         }
915
916         return tp;
917 }
918
919 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
920                                     unsigned int var_ref_idx);
921
922 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
923                                unsigned int var_ref_idx)
924 {
925         struct tracepoint *tp = event->tp;
926
927         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
928                 struct tracepoint_func *probe_func_ptr;
929                 synth_probe_func_t probe_func;
930                 void *__data;
931
932                 if (!(cpu_online(raw_smp_processor_id())))
933                         return;
934
935                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
936                 if (probe_func_ptr) {
937                         do {
938                                 probe_func = probe_func_ptr->func;
939                                 __data = probe_func_ptr->data;
940                                 probe_func(__data, var_ref_vals, var_ref_idx);
941                         } while ((++probe_func_ptr)->func);
942                 }
943         }
944 }
945
946 static struct synth_event *find_synth_event(const char *name)
947 {
948         struct dyn_event *pos;
949         struct synth_event *event;
950
951         for_each_dyn_event(pos) {
952                 if (!is_synth_event(pos))
953                         continue;
954                 event = to_synth_event(pos);
955                 if (strcmp(event->name, name) == 0)
956                         return event;
957         }
958
959         return NULL;
960 }
961
962 static int register_synth_event(struct synth_event *event)
963 {
964         struct trace_event_call *call = &event->call;
965         int ret = 0;
966
967         event->call.class = &event->class;
968         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
969         if (!event->class.system) {
970                 ret = -ENOMEM;
971                 goto out;
972         }
973
974         event->tp = alloc_synth_tracepoint(event->name);
975         if (IS_ERR(event->tp)) {
976                 ret = PTR_ERR(event->tp);
977                 event->tp = NULL;
978                 goto out;
979         }
980
981         INIT_LIST_HEAD(&call->class->fields);
982         call->event.funcs = &synth_event_funcs;
983         call->class->define_fields = synth_event_define_fields;
984
985         ret = register_trace_event(&call->event);
986         if (!ret) {
987                 ret = -ENODEV;
988                 goto out;
989         }
990         call->flags = TRACE_EVENT_FL_TRACEPOINT;
991         call->class->reg = trace_event_reg;
992         call->class->probe = trace_event_raw_event_synth;
993         call->data = event;
994         call->tp = event->tp;
995
996         ret = trace_add_event_call(call);
997         if (ret) {
998                 pr_warn("Failed to register synthetic event: %s\n",
999                         trace_event_name(call));
1000                 goto err;
1001         }
1002
1003         ret = set_synth_event_print_fmt(call);
1004         if (ret < 0) {
1005                 trace_remove_event_call(call);
1006                 goto err;
1007         }
1008  out:
1009         return ret;
1010  err:
1011         unregister_trace_event(&call->event);
1012         goto out;
1013 }
1014
1015 static int unregister_synth_event(struct synth_event *event)
1016 {
1017         struct trace_event_call *call = &event->call;
1018         int ret;
1019
1020         ret = trace_remove_event_call(call);
1021
1022         return ret;
1023 }
1024
1025 static void free_synth_event(struct synth_event *event)
1026 {
1027         unsigned int i;
1028
1029         if (!event)
1030                 return;
1031
1032         for (i = 0; i < event->n_fields; i++)
1033                 free_synth_field(event->fields[i]);
1034
1035         kfree(event->fields);
1036         kfree(event->name);
1037         kfree(event->class.system);
1038         free_synth_tracepoint(event->tp);
1039         free_synth_event_print_fmt(&event->call);
1040         kfree(event);
1041 }
1042
1043 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1044                                              struct synth_field **fields)
1045 {
1046         struct synth_event *event;
1047         unsigned int i;
1048
1049         event = kzalloc(sizeof(*event), GFP_KERNEL);
1050         if (!event) {
1051                 event = ERR_PTR(-ENOMEM);
1052                 goto out;
1053         }
1054
1055         event->name = kstrdup(name, GFP_KERNEL);
1056         if (!event->name) {
1057                 kfree(event);
1058                 event = ERR_PTR(-ENOMEM);
1059                 goto out;
1060         }
1061
1062         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1063         if (!event->fields) {
1064                 free_synth_event(event);
1065                 event = ERR_PTR(-ENOMEM);
1066                 goto out;
1067         }
1068
1069         dyn_event_init(&event->devent, &synth_event_ops);
1070
1071         for (i = 0; i < n_fields; i++)
1072                 event->fields[i] = fields[i];
1073
1074         event->n_fields = n_fields;
1075  out:
1076         return event;
1077 }
1078
1079 static void action_trace(struct hist_trigger_data *hist_data,
1080                          struct tracing_map_elt *elt, void *rec,
1081                          struct ring_buffer_event *rbe,
1082                          struct action_data *data, u64 *var_ref_vals)
1083 {
1084         struct synth_event *event = data->onmatch.synth_event;
1085
1086         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
1087 }
1088
1089 struct hist_var_data {
1090         struct list_head list;
1091         struct hist_trigger_data *hist_data;
1092 };
1093
1094 static int __create_synth_event(int argc, const char *name, const char **argv)
1095 {
1096         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1097         struct synth_event *event = NULL;
1098         int i, consumed = 0, n_fields = 0, ret = 0;
1099
1100         /*
1101          * Argument syntax:
1102          *  - Add synthetic event: <event_name> field[;field] ...
1103          *  - Remove synthetic event: !<event_name> field[;field] ...
1104          *      where 'field' = type field_name
1105          */
1106
1107         if (name[0] == '\0' || argc < 1)
1108                 return -EINVAL;
1109
1110         mutex_lock(&event_mutex);
1111
1112         event = find_synth_event(name);
1113         if (event) {
1114                 ret = -EEXIST;
1115                 goto out;
1116         }
1117
1118         for (i = 0; i < argc - 1; i++) {
1119                 if (strcmp(argv[i], ";") == 0)
1120                         continue;
1121                 if (n_fields == SYNTH_FIELDS_MAX) {
1122                         ret = -EINVAL;
1123                         goto err;
1124                 }
1125
1126                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1127                 if (IS_ERR(field)) {
1128                         ret = PTR_ERR(field);
1129                         goto err;
1130                 }
1131                 fields[n_fields++] = field;
1132                 i += consumed - 1;
1133         }
1134
1135         if (i < argc && strcmp(argv[i], ";") != 0) {
1136                 ret = -EINVAL;
1137                 goto err;
1138         }
1139
1140         event = alloc_synth_event(name, n_fields, fields);
1141         if (IS_ERR(event)) {
1142                 ret = PTR_ERR(event);
1143                 event = NULL;
1144                 goto err;
1145         }
1146         ret = register_synth_event(event);
1147         if (!ret)
1148                 dyn_event_add(&event->devent);
1149         else
1150                 free_synth_event(event);
1151  out:
1152         mutex_unlock(&event_mutex);
1153
1154         return ret;
1155  err:
1156         for (i = 0; i < n_fields; i++)
1157                 free_synth_field(fields[i]);
1158
1159         goto out;
1160 }
1161
1162 static int create_or_delete_synth_event(int argc, char **argv)
1163 {
1164         const char *name = argv[0];
1165         struct synth_event *event = NULL;
1166         int ret;
1167
1168         /* trace_run_command() ensures argc != 0 */
1169         if (name[0] == '!') {
1170                 mutex_lock(&event_mutex);
1171                 event = find_synth_event(name + 1);
1172                 if (event) {
1173                         if (event->ref)
1174                                 ret = -EBUSY;
1175                         else {
1176                                 ret = unregister_synth_event(event);
1177                                 if (!ret) {
1178                                         dyn_event_remove(&event->devent);
1179                                         free_synth_event(event);
1180                                 }
1181                         }
1182                 } else
1183                         ret = -ENOENT;
1184                 mutex_unlock(&event_mutex);
1185                 return ret;
1186         }
1187
1188         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1189         return ret == -ECANCELED ? -EINVAL : ret;
1190 }
1191
1192 static int synth_event_create(int argc, const char **argv)
1193 {
1194         const char *name = argv[0];
1195         int len;
1196
1197         if (name[0] != 's' || name[1] != ':')
1198                 return -ECANCELED;
1199         name += 2;
1200
1201         /* This interface accepts group name prefix */
1202         if (strchr(name, '/')) {
1203                 len = sizeof(SYNTH_SYSTEM "/") - 1;
1204                 if (strncmp(name, SYNTH_SYSTEM "/", len))
1205                         return -EINVAL;
1206                 name += len;
1207         }
1208         return __create_synth_event(argc - 1, name, argv + 1);
1209 }
1210
1211 static int synth_event_release(struct dyn_event *ev)
1212 {
1213         struct synth_event *event = to_synth_event(ev);
1214         int ret;
1215
1216         if (event->ref)
1217                 return -EBUSY;
1218
1219         ret = unregister_synth_event(event);
1220         if (ret)
1221                 return ret;
1222
1223         dyn_event_remove(ev);
1224         free_synth_event(event);
1225         return 0;
1226 }
1227
1228 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1229 {
1230         struct synth_field *field;
1231         unsigned int i;
1232
1233         seq_printf(m, "%s\t", event->name);
1234
1235         for (i = 0; i < event->n_fields; i++) {
1236                 field = event->fields[i];
1237
1238                 /* parameter values */
1239                 seq_printf(m, "%s %s%s", field->type, field->name,
1240                            i == event->n_fields - 1 ? "" : "; ");
1241         }
1242
1243         seq_putc(m, '\n');
1244
1245         return 0;
1246 }
1247
1248 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1249 {
1250         struct synth_event *event = to_synth_event(ev);
1251
1252         seq_printf(m, "s:%s/", event->class.system);
1253
1254         return __synth_event_show(m, event);
1255 }
1256
1257 static int synth_events_seq_show(struct seq_file *m, void *v)
1258 {
1259         struct dyn_event *ev = v;
1260
1261         if (!is_synth_event(ev))
1262                 return 0;
1263
1264         return __synth_event_show(m, to_synth_event(ev));
1265 }
1266
1267 static const struct seq_operations synth_events_seq_op = {
1268         .start  = dyn_event_seq_start,
1269         .next   = dyn_event_seq_next,
1270         .stop   = dyn_event_seq_stop,
1271         .show   = synth_events_seq_show,
1272 };
1273
1274 static int synth_events_open(struct inode *inode, struct file *file)
1275 {
1276         int ret;
1277
1278         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1279                 ret = dyn_events_release_all(&synth_event_ops);
1280                 if (ret < 0)
1281                         return ret;
1282         }
1283
1284         return seq_open(file, &synth_events_seq_op);
1285 }
1286
1287 static ssize_t synth_events_write(struct file *file,
1288                                   const char __user *buffer,
1289                                   size_t count, loff_t *ppos)
1290 {
1291         return trace_parse_run_command(file, buffer, count, ppos,
1292                                        create_or_delete_synth_event);
1293 }
1294
1295 static const struct file_operations synth_events_fops = {
1296         .open           = synth_events_open,
1297         .write          = synth_events_write,
1298         .read           = seq_read,
1299         .llseek         = seq_lseek,
1300         .release        = seq_release,
1301 };
1302
1303 static u64 hist_field_timestamp(struct hist_field *hist_field,
1304                                 struct tracing_map_elt *elt,
1305                                 struct ring_buffer_event *rbe,
1306                                 void *event)
1307 {
1308         struct hist_trigger_data *hist_data = hist_field->hist_data;
1309         struct trace_array *tr = hist_data->event_file->tr;
1310
1311         u64 ts = ring_buffer_event_time_stamp(rbe);
1312
1313         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1314                 ts = ns2usecs(ts);
1315
1316         return ts;
1317 }
1318
1319 static u64 hist_field_cpu(struct hist_field *hist_field,
1320                           struct tracing_map_elt *elt,
1321                           struct ring_buffer_event *rbe,
1322                           void *event)
1323 {
1324         int cpu = smp_processor_id();
1325
1326         return cpu;
1327 }
1328
1329 /**
1330  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1331  * @hist_field: The VAR_REF field to check
1332  * @var_data: The hist trigger that owns the variable
1333  * @var_idx: The trigger variable identifier
1334  *
1335  * Check the given VAR_REF field to see whether or not it references
1336  * the given variable associated with the given trigger.
1337  *
1338  * Return: The VAR_REF field if it does reference the variable, NULL if not
1339  */
1340 static struct hist_field *
1341 check_field_for_var_ref(struct hist_field *hist_field,
1342                         struct hist_trigger_data *var_data,
1343                         unsigned int var_idx)
1344 {
1345         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1346
1347         if (hist_field && hist_field->var.idx == var_idx &&
1348             hist_field->var.hist_data == var_data)
1349                 return hist_field;
1350
1351         return NULL;
1352 }
1353
1354 /**
1355  * find_var_ref - Check if a trigger has a reference to a trigger variable
1356  * @hist_data: The hist trigger that might have a reference to the variable
1357  * @var_data: The hist trigger that owns the variable
1358  * @var_idx: The trigger variable identifier
1359  *
1360  * Check the list of var_refs[] on the first hist trigger to see
1361  * whether any of them are references to the variable on the second
1362  * trigger.
1363  *
1364  * Return: The VAR_REF field referencing the variable if so, NULL if not
1365  */
1366 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1367                                        struct hist_trigger_data *var_data,
1368                                        unsigned int var_idx)
1369 {
1370         struct hist_field *hist_field;
1371         unsigned int i;
1372
1373         for (i = 0; i < hist_data->n_var_refs; i++) {
1374                 hist_field = hist_data->var_refs[i];
1375                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1376                         return hist_field;
1377         }
1378
1379         return NULL;
1380 }
1381
1382 /**
1383  * find_any_var_ref - Check if there is a reference to a given trigger variable
1384  * @hist_data: The hist trigger
1385  * @var_idx: The trigger variable identifier
1386  *
1387  * Check to see whether the given variable is currently referenced by
1388  * any other trigger.
1389  *
1390  * The trigger the variable is defined on is explicitly excluded - the
1391  * assumption being that a self-reference doesn't prevent a trigger
1392  * from being removed.
1393  *
1394  * Return: The VAR_REF field referencing the variable if so, NULL if not
1395  */
1396 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1397                                            unsigned int var_idx)
1398 {
1399         struct trace_array *tr = hist_data->event_file->tr;
1400         struct hist_field *found = NULL;
1401         struct hist_var_data *var_data;
1402
1403         list_for_each_entry(var_data, &tr->hist_vars, list) {
1404                 if (var_data->hist_data == hist_data)
1405                         continue;
1406                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1407                 if (found)
1408                         break;
1409         }
1410
1411         return found;
1412 }
1413
1414 /**
1415  * check_var_refs - Check if there is a reference to any of trigger's variables
1416  * @hist_data: The hist trigger
1417  *
1418  * A trigger can define one or more variables.  If any one of them is
1419  * currently referenced by any other trigger, this function will
1420  * determine that.
1421
1422  * Typically used to determine whether or not a trigger can be removed
1423  * - if there are any references to a trigger's variables, it cannot.
1424  *
1425  * Return: True if there is a reference to any of trigger's variables
1426  */
1427 static bool check_var_refs(struct hist_trigger_data *hist_data)
1428 {
1429         struct hist_field *field;
1430         bool found = false;
1431         int i;
1432
1433         for_each_hist_field(i, hist_data) {
1434                 field = hist_data->fields[i];
1435                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1436                         if (find_any_var_ref(hist_data, field->var.idx)) {
1437                                 found = true;
1438                                 break;
1439                         }
1440                 }
1441         }
1442
1443         return found;
1444 }
1445
1446 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1447 {
1448         struct trace_array *tr = hist_data->event_file->tr;
1449         struct hist_var_data *var_data, *found = NULL;
1450
1451         list_for_each_entry(var_data, &tr->hist_vars, list) {
1452                 if (var_data->hist_data == hist_data) {
1453                         found = var_data;
1454                         break;
1455                 }
1456         }
1457
1458         return found;
1459 }
1460
1461 static bool field_has_hist_vars(struct hist_field *hist_field,
1462                                 unsigned int level)
1463 {
1464         int i;
1465
1466         if (level > 3)
1467                 return false;
1468
1469         if (!hist_field)
1470                 return false;
1471
1472         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1473             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1474                 return true;
1475
1476         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1477                 struct hist_field *operand;
1478
1479                 operand = hist_field->operands[i];
1480                 if (field_has_hist_vars(operand, level + 1))
1481                         return true;
1482         }
1483
1484         return false;
1485 }
1486
1487 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1488 {
1489         struct hist_field *hist_field;
1490         int i;
1491
1492         for_each_hist_field(i, hist_data) {
1493                 hist_field = hist_data->fields[i];
1494                 if (field_has_hist_vars(hist_field, 0))
1495                         return true;
1496         }
1497
1498         return false;
1499 }
1500
1501 static int save_hist_vars(struct hist_trigger_data *hist_data)
1502 {
1503         struct trace_array *tr = hist_data->event_file->tr;
1504         struct hist_var_data *var_data;
1505
1506         var_data = find_hist_vars(hist_data);
1507         if (var_data)
1508                 return 0;
1509
1510         if (trace_array_get(tr) < 0)
1511                 return -ENODEV;
1512
1513         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1514         if (!var_data) {
1515                 trace_array_put(tr);
1516                 return -ENOMEM;
1517         }
1518
1519         var_data->hist_data = hist_data;
1520         list_add(&var_data->list, &tr->hist_vars);
1521
1522         return 0;
1523 }
1524
1525 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1526 {
1527         struct trace_array *tr = hist_data->event_file->tr;
1528         struct hist_var_data *var_data;
1529
1530         var_data = find_hist_vars(hist_data);
1531         if (!var_data)
1532                 return;
1533
1534         if (WARN_ON(check_var_refs(hist_data)))
1535                 return;
1536
1537         list_del(&var_data->list);
1538
1539         kfree(var_data);
1540
1541         trace_array_put(tr);
1542 }
1543
1544 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1545                                          const char *var_name)
1546 {
1547         struct hist_field *hist_field, *found = NULL;
1548         int i;
1549
1550         for_each_hist_field(i, hist_data) {
1551                 hist_field = hist_data->fields[i];
1552                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1553                     strcmp(hist_field->var.name, var_name) == 0) {
1554                         found = hist_field;
1555                         break;
1556                 }
1557         }
1558
1559         return found;
1560 }
1561
1562 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1563                                    struct trace_event_file *file,
1564                                    const char *var_name)
1565 {
1566         struct hist_trigger_data *test_data;
1567         struct event_trigger_data *test;
1568         struct hist_field *hist_field;
1569
1570         hist_field = find_var_field(hist_data, var_name);
1571         if (hist_field)
1572                 return hist_field;
1573
1574         list_for_each_entry_rcu(test, &file->triggers, list) {
1575                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1576                         test_data = test->private_data;
1577                         hist_field = find_var_field(test_data, var_name);
1578                         if (hist_field)
1579                                 return hist_field;
1580                 }
1581         }
1582
1583         return NULL;
1584 }
1585
1586 static struct trace_event_file *find_var_file(struct trace_array *tr,
1587                                               char *system,
1588                                               char *event_name,
1589                                               char *var_name)
1590 {
1591         struct hist_trigger_data *var_hist_data;
1592         struct hist_var_data *var_data;
1593         struct trace_event_file *file, *found = NULL;
1594
1595         if (system)
1596                 return find_event_file(tr, system, event_name);
1597
1598         list_for_each_entry(var_data, &tr->hist_vars, list) {
1599                 var_hist_data = var_data->hist_data;
1600                 file = var_hist_data->event_file;
1601                 if (file == found)
1602                         continue;
1603
1604                 if (find_var_field(var_hist_data, var_name)) {
1605                         if (found) {
1606                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1607                                 return NULL;
1608                         }
1609
1610                         found = file;
1611                 }
1612         }
1613
1614         return found;
1615 }
1616
1617 static struct hist_field *find_file_var(struct trace_event_file *file,
1618                                         const char *var_name)
1619 {
1620         struct hist_trigger_data *test_data;
1621         struct event_trigger_data *test;
1622         struct hist_field *hist_field;
1623
1624         list_for_each_entry_rcu(test, &file->triggers, list) {
1625                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1626                         test_data = test->private_data;
1627                         hist_field = find_var_field(test_data, var_name);
1628                         if (hist_field)
1629                                 return hist_field;
1630                 }
1631         }
1632
1633         return NULL;
1634 }
1635
1636 static struct hist_field *
1637 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1638 {
1639         struct trace_array *tr = hist_data->event_file->tr;
1640         struct hist_field *hist_field, *found = NULL;
1641         struct trace_event_file *file;
1642         unsigned int i;
1643
1644         for (i = 0; i < hist_data->n_actions; i++) {
1645                 struct action_data *data = hist_data->actions[i];
1646
1647                 if (data->fn == action_trace) {
1648                         char *system = data->onmatch.match_event_system;
1649                         char *event_name = data->onmatch.match_event;
1650
1651                         file = find_var_file(tr, system, event_name, var_name);
1652                         if (!file)
1653                                 continue;
1654                         hist_field = find_file_var(file, var_name);
1655                         if (hist_field) {
1656                                 if (found) {
1657                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1658                                         return ERR_PTR(-EINVAL);
1659                                 }
1660
1661                                 found = hist_field;
1662                         }
1663                 }
1664         }
1665         return found;
1666 }
1667
1668 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1669                                          char *system,
1670                                          char *event_name,
1671                                          char *var_name)
1672 {
1673         struct trace_array *tr = hist_data->event_file->tr;
1674         struct hist_field *hist_field = NULL;
1675         struct trace_event_file *file;
1676
1677         if (!system || !event_name) {
1678                 hist_field = find_match_var(hist_data, var_name);
1679                 if (IS_ERR(hist_field))
1680                         return NULL;
1681                 if (hist_field)
1682                         return hist_field;
1683         }
1684
1685         file = find_var_file(tr, system, event_name, var_name);
1686         if (!file)
1687                 return NULL;
1688
1689         hist_field = find_file_var(file, var_name);
1690
1691         return hist_field;
1692 }
1693
1694 struct hist_elt_data {
1695         char *comm;
1696         u64 *var_ref_vals;
1697         char *field_var_str[SYNTH_FIELDS_MAX];
1698 };
1699
1700 static u64 hist_field_var_ref(struct hist_field *hist_field,
1701                               struct tracing_map_elt *elt,
1702                               struct ring_buffer_event *rbe,
1703                               void *event)
1704 {
1705         struct hist_elt_data *elt_data;
1706         u64 var_val = 0;
1707
1708         elt_data = elt->private_data;
1709         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1710
1711         return var_val;
1712 }
1713
1714 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1715                              u64 *var_ref_vals, bool self)
1716 {
1717         struct hist_trigger_data *var_data;
1718         struct tracing_map_elt *var_elt;
1719         struct hist_field *hist_field;
1720         unsigned int i, var_idx;
1721         bool resolved = true;
1722         u64 var_val = 0;
1723
1724         for (i = 0; i < hist_data->n_var_refs; i++) {
1725                 hist_field = hist_data->var_refs[i];
1726                 var_idx = hist_field->var.idx;
1727                 var_data = hist_field->var.hist_data;
1728
1729                 if (var_data == NULL) {
1730                         resolved = false;
1731                         break;
1732                 }
1733
1734                 if ((self && var_data != hist_data) ||
1735                     (!self && var_data == hist_data))
1736                         continue;
1737
1738                 var_elt = tracing_map_lookup(var_data->map, key);
1739                 if (!var_elt) {
1740                         resolved = false;
1741                         break;
1742                 }
1743
1744                 if (!tracing_map_var_set(var_elt, var_idx)) {
1745                         resolved = false;
1746                         break;
1747                 }
1748
1749                 if (self || !hist_field->read_once)
1750                         var_val = tracing_map_read_var(var_elt, var_idx);
1751                 else
1752                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1753
1754                 var_ref_vals[i] = var_val;
1755         }
1756
1757         return resolved;
1758 }
1759
1760 static const char *hist_field_name(struct hist_field *field,
1761                                    unsigned int level)
1762 {
1763         const char *field_name = "";
1764
1765         if (level > 1)
1766                 return field_name;
1767
1768         if (field->field)
1769                 field_name = field->field->name;
1770         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1771                  field->flags & HIST_FIELD_FL_ALIAS)
1772                 field_name = hist_field_name(field->operands[0], ++level);
1773         else if (field->flags & HIST_FIELD_FL_CPU)
1774                 field_name = "cpu";
1775         else if (field->flags & HIST_FIELD_FL_EXPR ||
1776                  field->flags & HIST_FIELD_FL_VAR_REF) {
1777                 if (field->system) {
1778                         static char full_name[MAX_FILTER_STR_VAL];
1779
1780                         strcat(full_name, field->system);
1781                         strcat(full_name, ".");
1782                         strcat(full_name, field->event_name);
1783                         strcat(full_name, ".");
1784                         strcat(full_name, field->name);
1785                         field_name = full_name;
1786                 } else
1787                         field_name = field->name;
1788         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1789                 field_name = "common_timestamp";
1790
1791         if (field_name == NULL)
1792                 field_name = "";
1793
1794         return field_name;
1795 }
1796
1797 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1798 {
1799         hist_field_fn_t fn = NULL;
1800
1801         switch (field_size) {
1802         case 8:
1803                 if (field_is_signed)
1804                         fn = hist_field_s64;
1805                 else
1806                         fn = hist_field_u64;
1807                 break;
1808         case 4:
1809                 if (field_is_signed)
1810                         fn = hist_field_s32;
1811                 else
1812                         fn = hist_field_u32;
1813                 break;
1814         case 2:
1815                 if (field_is_signed)
1816                         fn = hist_field_s16;
1817                 else
1818                         fn = hist_field_u16;
1819                 break;
1820         case 1:
1821                 if (field_is_signed)
1822                         fn = hist_field_s8;
1823                 else
1824                         fn = hist_field_u8;
1825                 break;
1826         }
1827
1828         return fn;
1829 }
1830
1831 static int parse_map_size(char *str)
1832 {
1833         unsigned long size, map_bits;
1834         int ret;
1835
1836         strsep(&str, "=");
1837         if (!str) {
1838                 ret = -EINVAL;
1839                 goto out;
1840         }
1841
1842         ret = kstrtoul(str, 0, &size);
1843         if (ret)
1844                 goto out;
1845
1846         map_bits = ilog2(roundup_pow_of_two(size));
1847         if (map_bits < TRACING_MAP_BITS_MIN ||
1848             map_bits > TRACING_MAP_BITS_MAX)
1849                 ret = -EINVAL;
1850         else
1851                 ret = map_bits;
1852  out:
1853         return ret;
1854 }
1855
1856 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1857 {
1858         unsigned int i;
1859
1860         if (!attrs)
1861                 return;
1862
1863         for (i = 0; i < attrs->n_assignments; i++)
1864                 kfree(attrs->assignment_str[i]);
1865
1866         for (i = 0; i < attrs->n_actions; i++)
1867                 kfree(attrs->action_str[i]);
1868
1869         kfree(attrs->name);
1870         kfree(attrs->sort_key_str);
1871         kfree(attrs->keys_str);
1872         kfree(attrs->vals_str);
1873         kfree(attrs->clock);
1874         kfree(attrs);
1875 }
1876
1877 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1878 {
1879         int ret = -EINVAL;
1880
1881         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1882                 return ret;
1883
1884         if ((str_has_prefix(str, "onmatch(")) ||
1885             (str_has_prefix(str, "onmax("))) {
1886                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1887                 if (!attrs->action_str[attrs->n_actions]) {
1888                         ret = -ENOMEM;
1889                         return ret;
1890                 }
1891                 attrs->n_actions++;
1892                 ret = 0;
1893         }
1894
1895         return ret;
1896 }
1897
1898 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1899 {
1900         int ret = 0;
1901
1902         if ((str_has_prefix(str, "key=")) ||
1903             (str_has_prefix(str, "keys="))) {
1904                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1905                 if (!attrs->keys_str) {
1906                         ret = -ENOMEM;
1907                         goto out;
1908                 }
1909         } else if ((str_has_prefix(str, "val=")) ||
1910                    (str_has_prefix(str, "vals=")) ||
1911                    (str_has_prefix(str, "values="))) {
1912                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1913                 if (!attrs->vals_str) {
1914                         ret = -ENOMEM;
1915                         goto out;
1916                 }
1917         } else if (str_has_prefix(str, "sort=")) {
1918                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1919                 if (!attrs->sort_key_str) {
1920                         ret = -ENOMEM;
1921                         goto out;
1922                 }
1923         } else if (str_has_prefix(str, "name=")) {
1924                 attrs->name = kstrdup(str, GFP_KERNEL);
1925                 if (!attrs->name) {
1926                         ret = -ENOMEM;
1927                         goto out;
1928                 }
1929         } else if (str_has_prefix(str, "clock=")) {
1930                 strsep(&str, "=");
1931                 if (!str) {
1932                         ret = -EINVAL;
1933                         goto out;
1934                 }
1935
1936                 str = strstrip(str);
1937                 attrs->clock = kstrdup(str, GFP_KERNEL);
1938                 if (!attrs->clock) {
1939                         ret = -ENOMEM;
1940                         goto out;
1941                 }
1942         } else if (str_has_prefix(str, "size=")) {
1943                 int map_bits = parse_map_size(str);
1944
1945                 if (map_bits < 0) {
1946                         ret = map_bits;
1947                         goto out;
1948                 }
1949                 attrs->map_bits = map_bits;
1950         } else {
1951                 char *assignment;
1952
1953                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1954                         hist_err("Too many variables defined: ", str);
1955                         ret = -EINVAL;
1956                         goto out;
1957                 }
1958
1959                 assignment = kstrdup(str, GFP_KERNEL);
1960                 if (!assignment) {
1961                         ret = -ENOMEM;
1962                         goto out;
1963                 }
1964
1965                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1966         }
1967  out:
1968         return ret;
1969 }
1970
1971 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1972 {
1973         struct hist_trigger_attrs *attrs;
1974         int ret = 0;
1975
1976         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1977         if (!attrs)
1978                 return ERR_PTR(-ENOMEM);
1979
1980         while (trigger_str) {
1981                 char *str = strsep(&trigger_str, ":");
1982
1983                 if (strchr(str, '=')) {
1984                         ret = parse_assignment(str, attrs);
1985                         if (ret)
1986                                 goto free;
1987                 } else if (strcmp(str, "pause") == 0)
1988                         attrs->pause = true;
1989                 else if ((strcmp(str, "cont") == 0) ||
1990                          (strcmp(str, "continue") == 0))
1991                         attrs->cont = true;
1992                 else if (strcmp(str, "clear") == 0)
1993                         attrs->clear = true;
1994                 else {
1995                         ret = parse_action(str, attrs);
1996                         if (ret)
1997                                 goto free;
1998                 }
1999         }
2000
2001         if (!attrs->keys_str) {
2002                 ret = -EINVAL;
2003                 goto free;
2004         }
2005
2006         if (!attrs->clock) {
2007                 attrs->clock = kstrdup("global", GFP_KERNEL);
2008                 if (!attrs->clock) {
2009                         ret = -ENOMEM;
2010                         goto free;
2011                 }
2012         }
2013
2014         return attrs;
2015  free:
2016         destroy_hist_trigger_attrs(attrs);
2017
2018         return ERR_PTR(ret);
2019 }
2020
2021 static inline void save_comm(char *comm, struct task_struct *task)
2022 {
2023         if (!task->pid) {
2024                 strcpy(comm, "<idle>");
2025                 return;
2026         }
2027
2028         if (WARN_ON_ONCE(task->pid < 0)) {
2029                 strcpy(comm, "<XXX>");
2030                 return;
2031         }
2032
2033         memcpy(comm, task->comm, TASK_COMM_LEN);
2034 }
2035
2036 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2037 {
2038         unsigned int i;
2039
2040         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2041                 kfree(elt_data->field_var_str[i]);
2042
2043         kfree(elt_data->comm);
2044         kfree(elt_data);
2045 }
2046
2047 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2048 {
2049         struct hist_elt_data *elt_data = elt->private_data;
2050
2051         hist_elt_data_free(elt_data);
2052 }
2053
2054 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2055 {
2056         struct hist_trigger_data *hist_data = elt->map->private_data;
2057         unsigned int size = TASK_COMM_LEN;
2058         struct hist_elt_data *elt_data;
2059         struct hist_field *key_field;
2060         unsigned int i, n_str;
2061
2062         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2063         if (!elt_data)
2064                 return -ENOMEM;
2065
2066         for_each_hist_key_field(i, hist_data) {
2067                 key_field = hist_data->fields[i];
2068
2069                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2070                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2071                         if (!elt_data->comm) {
2072                                 kfree(elt_data);
2073                                 return -ENOMEM;
2074                         }
2075                         break;
2076                 }
2077         }
2078
2079         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
2080
2081         size = STR_VAR_LEN_MAX;
2082
2083         for (i = 0; i < n_str; i++) {
2084                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2085                 if (!elt_data->field_var_str[i]) {
2086                         hist_elt_data_free(elt_data);
2087                         return -ENOMEM;
2088                 }
2089         }
2090
2091         elt->private_data = elt_data;
2092
2093         return 0;
2094 }
2095
2096 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2097 {
2098         struct hist_elt_data *elt_data = elt->private_data;
2099
2100         if (elt_data->comm)
2101                 save_comm(elt_data->comm, current);
2102 }
2103
2104 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2105         .elt_alloc      = hist_trigger_elt_data_alloc,
2106         .elt_free       = hist_trigger_elt_data_free,
2107         .elt_init       = hist_trigger_elt_data_init,
2108 };
2109
2110 static const char *get_hist_field_flags(struct hist_field *hist_field)
2111 {
2112         const char *flags_str = NULL;
2113
2114         if (hist_field->flags & HIST_FIELD_FL_HEX)
2115                 flags_str = "hex";
2116         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2117                 flags_str = "sym";
2118         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2119                 flags_str = "sym-offset";
2120         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2121                 flags_str = "execname";
2122         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2123                 flags_str = "syscall";
2124         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2125                 flags_str = "log2";
2126         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2127                 flags_str = "usecs";
2128
2129         return flags_str;
2130 }
2131
2132 static void expr_field_str(struct hist_field *field, char *expr)
2133 {
2134         if (field->flags & HIST_FIELD_FL_VAR_REF)
2135                 strcat(expr, "$");
2136
2137         strcat(expr, hist_field_name(field, 0));
2138
2139         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2140                 const char *flags_str = get_hist_field_flags(field);
2141
2142                 if (flags_str) {
2143                         strcat(expr, ".");
2144                         strcat(expr, flags_str);
2145                 }
2146         }
2147 }
2148
2149 static char *expr_str(struct hist_field *field, unsigned int level)
2150 {
2151         char *expr;
2152
2153         if (level > 1)
2154                 return NULL;
2155
2156         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2157         if (!expr)
2158                 return NULL;
2159
2160         if (!field->operands[0]) {
2161                 expr_field_str(field, expr);
2162                 return expr;
2163         }
2164
2165         if (field->operator == FIELD_OP_UNARY_MINUS) {
2166                 char *subexpr;
2167
2168                 strcat(expr, "-(");
2169                 subexpr = expr_str(field->operands[0], ++level);
2170                 if (!subexpr) {
2171                         kfree(expr);
2172                         return NULL;
2173                 }
2174                 strcat(expr, subexpr);
2175                 strcat(expr, ")");
2176
2177                 kfree(subexpr);
2178
2179                 return expr;
2180         }
2181
2182         expr_field_str(field->operands[0], expr);
2183
2184         switch (field->operator) {
2185         case FIELD_OP_MINUS:
2186                 strcat(expr, "-");
2187                 break;
2188         case FIELD_OP_PLUS:
2189                 strcat(expr, "+");
2190                 break;
2191         default:
2192                 kfree(expr);
2193                 return NULL;
2194         }
2195
2196         expr_field_str(field->operands[1], expr);
2197
2198         return expr;
2199 }
2200
2201 static int contains_operator(char *str)
2202 {
2203         enum field_op_id field_op = FIELD_OP_NONE;
2204         char *op;
2205
2206         op = strpbrk(str, "+-");
2207         if (!op)
2208                 return FIELD_OP_NONE;
2209
2210         switch (*op) {
2211         case '-':
2212                 if (*str == '-')
2213                         field_op = FIELD_OP_UNARY_MINUS;
2214                 else
2215                         field_op = FIELD_OP_MINUS;
2216                 break;
2217         case '+':
2218                 field_op = FIELD_OP_PLUS;
2219                 break;
2220         default:
2221                 break;
2222         }
2223
2224         return field_op;
2225 }
2226
2227 static void __destroy_hist_field(struct hist_field *hist_field)
2228 {
2229         kfree(hist_field->var.name);
2230         kfree(hist_field->name);
2231         kfree(hist_field->type);
2232
2233         kfree(hist_field);
2234 }
2235
2236 static void destroy_hist_field(struct hist_field *hist_field,
2237                                unsigned int level)
2238 {
2239         unsigned int i;
2240
2241         if (level > 3)
2242                 return;
2243
2244         if (!hist_field)
2245                 return;
2246
2247         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2248                 return; /* var refs will be destroyed separately */
2249
2250         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2251                 destroy_hist_field(hist_field->operands[i], level + 1);
2252
2253         __destroy_hist_field(hist_field);
2254 }
2255
2256 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2257                                             struct ftrace_event_field *field,
2258                                             unsigned long flags,
2259                                             char *var_name)
2260 {
2261         struct hist_field *hist_field;
2262
2263         if (field && is_function_field(field))
2264                 return NULL;
2265
2266         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2267         if (!hist_field)
2268                 return NULL;
2269
2270         hist_field->hist_data = hist_data;
2271
2272         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2273                 goto out; /* caller will populate */
2274
2275         if (flags & HIST_FIELD_FL_VAR_REF) {
2276                 hist_field->fn = hist_field_var_ref;
2277                 goto out;
2278         }
2279
2280         if (flags & HIST_FIELD_FL_HITCOUNT) {
2281                 hist_field->fn = hist_field_counter;
2282                 hist_field->size = sizeof(u64);
2283                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2284                 if (!hist_field->type)
2285                         goto free;
2286                 goto out;
2287         }
2288
2289         if (flags & HIST_FIELD_FL_STACKTRACE) {
2290                 hist_field->fn = hist_field_none;
2291                 goto out;
2292         }
2293
2294         if (flags & HIST_FIELD_FL_LOG2) {
2295                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2296                 hist_field->fn = hist_field_log2;
2297                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2298                 hist_field->size = hist_field->operands[0]->size;
2299                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2300                 if (!hist_field->type)
2301                         goto free;
2302                 goto out;
2303         }
2304
2305         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2306                 hist_field->fn = hist_field_timestamp;
2307                 hist_field->size = sizeof(u64);
2308                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2309                 if (!hist_field->type)
2310                         goto free;
2311                 goto out;
2312         }
2313
2314         if (flags & HIST_FIELD_FL_CPU) {
2315                 hist_field->fn = hist_field_cpu;
2316                 hist_field->size = sizeof(int);
2317                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2318                 if (!hist_field->type)
2319                         goto free;
2320                 goto out;
2321         }
2322
2323         if (WARN_ON_ONCE(!field))
2324                 goto out;
2325
2326         if (is_string_field(field)) {
2327                 flags |= HIST_FIELD_FL_STRING;
2328
2329                 hist_field->size = MAX_FILTER_STR_VAL;
2330                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2331                 if (!hist_field->type)
2332                         goto free;
2333
2334                 if (field->filter_type == FILTER_STATIC_STRING)
2335                         hist_field->fn = hist_field_string;
2336                 else if (field->filter_type == FILTER_DYN_STRING)
2337                         hist_field->fn = hist_field_dynstring;
2338                 else
2339                         hist_field->fn = hist_field_pstring;
2340         } else {
2341                 hist_field->size = field->size;
2342                 hist_field->is_signed = field->is_signed;
2343                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2344                 if (!hist_field->type)
2345                         goto free;
2346
2347                 hist_field->fn = select_value_fn(field->size,
2348                                                  field->is_signed);
2349                 if (!hist_field->fn) {
2350                         destroy_hist_field(hist_field, 0);
2351                         return NULL;
2352                 }
2353         }
2354  out:
2355         hist_field->field = field;
2356         hist_field->flags = flags;
2357
2358         if (var_name) {
2359                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2360                 if (!hist_field->var.name)
2361                         goto free;
2362         }
2363
2364         return hist_field;
2365  free:
2366         destroy_hist_field(hist_field, 0);
2367         return NULL;
2368 }
2369
2370 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2371 {
2372         unsigned int i;
2373
2374         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2375                 if (hist_data->fields[i]) {
2376                         destroy_hist_field(hist_data->fields[i], 0);
2377                         hist_data->fields[i] = NULL;
2378                 }
2379         }
2380
2381         for (i = 0; i < hist_data->n_var_refs; i++) {
2382                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2383                 __destroy_hist_field(hist_data->var_refs[i]);
2384                 hist_data->var_refs[i] = NULL;
2385         }
2386 }
2387
2388 static int init_var_ref(struct hist_field *ref_field,
2389                         struct hist_field *var_field,
2390                         char *system, char *event_name)
2391 {
2392         int err = 0;
2393
2394         ref_field->var.idx = var_field->var.idx;
2395         ref_field->var.hist_data = var_field->hist_data;
2396         ref_field->size = var_field->size;
2397         ref_field->is_signed = var_field->is_signed;
2398         ref_field->flags |= var_field->flags &
2399                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2400
2401         if (system) {
2402                 ref_field->system = kstrdup(system, GFP_KERNEL);
2403                 if (!ref_field->system)
2404                         return -ENOMEM;
2405         }
2406
2407         if (event_name) {
2408                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2409                 if (!ref_field->event_name) {
2410                         err = -ENOMEM;
2411                         goto free;
2412                 }
2413         }
2414
2415         if (var_field->var.name) {
2416                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2417                 if (!ref_field->name) {
2418                         err = -ENOMEM;
2419                         goto free;
2420                 }
2421         } else if (var_field->name) {
2422                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2423                 if (!ref_field->name) {
2424                         err = -ENOMEM;
2425                         goto free;
2426                 }
2427         }
2428
2429         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2430         if (!ref_field->type) {
2431                 err = -ENOMEM;
2432                 goto free;
2433         }
2434  out:
2435         return err;
2436  free:
2437         kfree(ref_field->system);
2438         kfree(ref_field->event_name);
2439         kfree(ref_field->name);
2440
2441         goto out;
2442 }
2443
2444 /**
2445  * create_var_ref - Create a variable reference and attach it to trigger
2446  * @hist_data: The trigger that will be referencing the variable
2447  * @var_field: The VAR field to create a reference to
2448  * @system: The optional system string
2449  * @event_name: The optional event_name string
2450  *
2451  * Given a variable hist_field, create a VAR_REF hist_field that
2452  * represents a reference to it.
2453  *
2454  * This function also adds the reference to the trigger that
2455  * now references the variable.
2456  *
2457  * Return: The VAR_REF field if successful, NULL if not
2458  */
2459 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2460                                          struct hist_field *var_field,
2461                                          char *system, char *event_name)
2462 {
2463         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2464         struct hist_field *ref_field;
2465
2466         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2467         if (ref_field) {
2468                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2469                         destroy_hist_field(ref_field, 0);
2470                         return NULL;
2471                 }
2472
2473                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2474                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2475         }
2476
2477         return ref_field;
2478 }
2479
2480 static bool is_var_ref(char *var_name)
2481 {
2482         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2483                 return false;
2484
2485         return true;
2486 }
2487
2488 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2489                                  char *var_name)
2490 {
2491         char *name, *field;
2492         unsigned int i;
2493
2494         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2495                 name = hist_data->attrs->var_defs.name[i];
2496
2497                 if (strcmp(var_name, name) == 0) {
2498                         field = hist_data->attrs->var_defs.expr[i];
2499                         if (contains_operator(field) || is_var_ref(field))
2500                                 continue;
2501                         return field;
2502                 }
2503         }
2504
2505         return NULL;
2506 }
2507
2508 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2509                                  char *system, char *event_name,
2510                                  char *var_name)
2511 {
2512         struct trace_event_call *call;
2513
2514         if (system && event_name) {
2515                 call = hist_data->event_file->event_call;
2516
2517                 if (strcmp(system, call->class->system) != 0)
2518                         return NULL;
2519
2520                 if (strcmp(event_name, trace_event_name(call)) != 0)
2521                         return NULL;
2522         }
2523
2524         if (!!system != !!event_name)
2525                 return NULL;
2526
2527         if (!is_var_ref(var_name))
2528                 return NULL;
2529
2530         var_name++;
2531
2532         return field_name_from_var(hist_data, var_name);
2533 }
2534
2535 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2536                                         char *system, char *event_name,
2537                                         char *var_name)
2538 {
2539         struct hist_field *var_field = NULL, *ref_field = NULL;
2540
2541         if (!is_var_ref(var_name))
2542                 return NULL;
2543
2544         var_name++;
2545
2546         var_field = find_event_var(hist_data, system, event_name, var_name);
2547         if (var_field)
2548                 ref_field = create_var_ref(hist_data, var_field,
2549                                            system, event_name);
2550
2551         if (!ref_field)
2552                 hist_err_event("Couldn't find variable: $",
2553                                system, event_name, var_name);
2554
2555         return ref_field;
2556 }
2557
2558 static struct ftrace_event_field *
2559 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2560             char *field_str, unsigned long *flags)
2561 {
2562         struct ftrace_event_field *field = NULL;
2563         char *field_name, *modifier, *str;
2564
2565         modifier = str = kstrdup(field_str, GFP_KERNEL);
2566         if (!modifier)
2567                 return ERR_PTR(-ENOMEM);
2568
2569         field_name = strsep(&modifier, ".");
2570         if (modifier) {
2571                 if (strcmp(modifier, "hex") == 0)
2572                         *flags |= HIST_FIELD_FL_HEX;
2573                 else if (strcmp(modifier, "sym") == 0)
2574                         *flags |= HIST_FIELD_FL_SYM;
2575                 else if (strcmp(modifier, "sym-offset") == 0)
2576                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2577                 else if ((strcmp(modifier, "execname") == 0) &&
2578                          (strcmp(field_name, "common_pid") == 0))
2579                         *flags |= HIST_FIELD_FL_EXECNAME;
2580                 else if (strcmp(modifier, "syscall") == 0)
2581                         *flags |= HIST_FIELD_FL_SYSCALL;
2582                 else if (strcmp(modifier, "log2") == 0)
2583                         *flags |= HIST_FIELD_FL_LOG2;
2584                 else if (strcmp(modifier, "usecs") == 0)
2585                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2586                 else {
2587                         hist_err("Invalid field modifier: ", modifier);
2588                         field = ERR_PTR(-EINVAL);
2589                         goto out;
2590                 }
2591         }
2592
2593         if (strcmp(field_name, "common_timestamp") == 0) {
2594                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2595                 hist_data->enable_timestamps = true;
2596                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2597                         hist_data->attrs->ts_in_usecs = true;
2598         } else if (strcmp(field_name, "cpu") == 0)
2599                 *flags |= HIST_FIELD_FL_CPU;
2600         else {
2601                 field = trace_find_event_field(file->event_call, field_name);
2602                 if (!field || !field->size) {
2603                         hist_err("Couldn't find field: ", field_name);
2604                         field = ERR_PTR(-EINVAL);
2605                         goto out;
2606                 }
2607         }
2608  out:
2609         kfree(str);
2610
2611         return field;
2612 }
2613
2614 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2615                                        struct hist_field *var_ref,
2616                                        char *var_name)
2617 {
2618         struct hist_field *alias = NULL;
2619         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2620
2621         alias = create_hist_field(hist_data, NULL, flags, var_name);
2622         if (!alias)
2623                 return NULL;
2624
2625         alias->fn = var_ref->fn;
2626         alias->operands[0] = var_ref;
2627
2628         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2629                 destroy_hist_field(alias, 0);
2630                 return NULL;
2631         }
2632
2633         return alias;
2634 }
2635
2636 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2637                                      struct trace_event_file *file, char *str,
2638                                      unsigned long *flags, char *var_name)
2639 {
2640         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2641         struct ftrace_event_field *field = NULL;
2642         struct hist_field *hist_field = NULL;
2643         int ret = 0;
2644
2645         s = strchr(str, '.');
2646         if (s) {
2647                 s = strchr(++s, '.');
2648                 if (s) {
2649                         ref_system = strsep(&str, ".");
2650                         if (!str) {
2651                                 ret = -EINVAL;
2652                                 goto out;
2653                         }
2654                         ref_event = strsep(&str, ".");
2655                         if (!str) {
2656                                 ret = -EINVAL;
2657                                 goto out;
2658                         }
2659                         ref_var = str;
2660                 }
2661         }
2662
2663         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2664         if (!s) {
2665                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2666                 if (hist_field) {
2667                         if (var_name) {
2668                                 hist_field = create_alias(hist_data, hist_field, var_name);
2669                                 if (!hist_field) {
2670                                         ret = -ENOMEM;
2671                                         goto out;
2672                                 }
2673                         }
2674                         return hist_field;
2675                 }
2676         } else
2677                 str = s;
2678
2679         field = parse_field(hist_data, file, str, flags);
2680         if (IS_ERR(field)) {
2681                 ret = PTR_ERR(field);
2682                 goto out;
2683         }
2684
2685         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2686         if (!hist_field) {
2687                 ret = -ENOMEM;
2688                 goto out;
2689         }
2690
2691         return hist_field;
2692  out:
2693         return ERR_PTR(ret);
2694 }
2695
2696 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2697                                      struct trace_event_file *file,
2698                                      char *str, unsigned long flags,
2699                                      char *var_name, unsigned int level);
2700
2701 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2702                                       struct trace_event_file *file,
2703                                       char *str, unsigned long flags,
2704                                       char *var_name, unsigned int level)
2705 {
2706         struct hist_field *operand1, *expr = NULL;
2707         unsigned long operand_flags;
2708         int ret = 0;
2709         char *s;
2710
2711         /* we support only -(xxx) i.e. explicit parens required */
2712
2713         if (level > 3) {
2714                 hist_err("Too many subexpressions (3 max): ", str);
2715                 ret = -EINVAL;
2716                 goto free;
2717         }
2718
2719         str++; /* skip leading '-' */
2720
2721         s = strchr(str, '(');
2722         if (s)
2723                 str++;
2724         else {
2725                 ret = -EINVAL;
2726                 goto free;
2727         }
2728
2729         s = strrchr(str, ')');
2730         if (s)
2731                 *s = '\0';
2732         else {
2733                 ret = -EINVAL; /* no closing ')' */
2734                 goto free;
2735         }
2736
2737         flags |= HIST_FIELD_FL_EXPR;
2738         expr = create_hist_field(hist_data, NULL, flags, var_name);
2739         if (!expr) {
2740                 ret = -ENOMEM;
2741                 goto free;
2742         }
2743
2744         operand_flags = 0;
2745         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2746         if (IS_ERR(operand1)) {
2747                 ret = PTR_ERR(operand1);
2748                 goto free;
2749         }
2750
2751         expr->flags |= operand1->flags &
2752                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2753         expr->fn = hist_field_unary_minus;
2754         expr->operands[0] = operand1;
2755         expr->operator = FIELD_OP_UNARY_MINUS;
2756         expr->name = expr_str(expr, 0);
2757         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2758         if (!expr->type) {
2759                 ret = -ENOMEM;
2760                 goto free;
2761         }
2762
2763         return expr;
2764  free:
2765         destroy_hist_field(expr, 0);
2766         return ERR_PTR(ret);
2767 }
2768
2769 static int check_expr_operands(struct hist_field *operand1,
2770                                struct hist_field *operand2)
2771 {
2772         unsigned long operand1_flags = operand1->flags;
2773         unsigned long operand2_flags = operand2->flags;
2774
2775         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2776             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2777                 struct hist_field *var;
2778
2779                 var = find_var_field(operand1->var.hist_data, operand1->name);
2780                 if (!var)
2781                         return -EINVAL;
2782                 operand1_flags = var->flags;
2783         }
2784
2785         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2786             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2787                 struct hist_field *var;
2788
2789                 var = find_var_field(operand2->var.hist_data, operand2->name);
2790                 if (!var)
2791                         return -EINVAL;
2792                 operand2_flags = var->flags;
2793         }
2794
2795         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2796             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2797                 hist_err("Timestamp units in expression don't match", NULL);
2798                 return -EINVAL;
2799         }
2800
2801         return 0;
2802 }
2803
2804 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2805                                      struct trace_event_file *file,
2806                                      char *str, unsigned long flags,
2807                                      char *var_name, unsigned int level)
2808 {
2809         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2810         unsigned long operand_flags;
2811         int field_op, ret = -EINVAL;
2812         char *sep, *operand1_str;
2813
2814         if (level > 3) {
2815                 hist_err("Too many subexpressions (3 max): ", str);
2816                 return ERR_PTR(-EINVAL);
2817         }
2818
2819         field_op = contains_operator(str);
2820
2821         if (field_op == FIELD_OP_NONE)
2822                 return parse_atom(hist_data, file, str, &flags, var_name);
2823
2824         if (field_op == FIELD_OP_UNARY_MINUS)
2825                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2826
2827         switch (field_op) {
2828         case FIELD_OP_MINUS:
2829                 sep = "-";
2830                 break;
2831         case FIELD_OP_PLUS:
2832                 sep = "+";
2833                 break;
2834         default:
2835                 goto free;
2836         }
2837
2838         operand1_str = strsep(&str, sep);
2839         if (!operand1_str || !str)
2840                 goto free;
2841
2842         operand_flags = 0;
2843         operand1 = parse_atom(hist_data, file, operand1_str,
2844                               &operand_flags, NULL);
2845         if (IS_ERR(operand1)) {
2846                 ret = PTR_ERR(operand1);
2847                 operand1 = NULL;
2848                 goto free;
2849         }
2850
2851         /* rest of string could be another expression e.g. b+c in a+b+c */
2852         operand_flags = 0;
2853         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2854         if (IS_ERR(operand2)) {
2855                 ret = PTR_ERR(operand2);
2856                 operand2 = NULL;
2857                 goto free;
2858         }
2859
2860         ret = check_expr_operands(operand1, operand2);
2861         if (ret)
2862                 goto free;
2863
2864         flags |= HIST_FIELD_FL_EXPR;
2865
2866         flags |= operand1->flags &
2867                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2868
2869         expr = create_hist_field(hist_data, NULL, flags, var_name);
2870         if (!expr) {
2871                 ret = -ENOMEM;
2872                 goto free;
2873         }
2874
2875         operand1->read_once = true;
2876         operand2->read_once = true;
2877
2878         expr->operands[0] = operand1;
2879         expr->operands[1] = operand2;
2880         expr->operator = field_op;
2881         expr->name = expr_str(expr, 0);
2882         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2883         if (!expr->type) {
2884                 ret = -ENOMEM;
2885                 goto free;
2886         }
2887
2888         switch (field_op) {
2889         case FIELD_OP_MINUS:
2890                 expr->fn = hist_field_minus;
2891                 break;
2892         case FIELD_OP_PLUS:
2893                 expr->fn = hist_field_plus;
2894                 break;
2895         default:
2896                 ret = -EINVAL;
2897                 goto free;
2898         }
2899
2900         return expr;
2901  free:
2902         destroy_hist_field(operand1, 0);
2903         destroy_hist_field(operand2, 0);
2904         destroy_hist_field(expr, 0);
2905
2906         return ERR_PTR(ret);
2907 }
2908
2909 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2910                                  struct trace_event_file *file)
2911 {
2912         struct event_trigger_data *test;
2913
2914         list_for_each_entry_rcu(test, &file->triggers, list) {
2915                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2916                         if (test->private_data == hist_data)
2917                                 return test->filter_str;
2918                 }
2919         }
2920
2921         return NULL;
2922 }
2923
2924 static struct event_command trigger_hist_cmd;
2925 static int event_hist_trigger_func(struct event_command *cmd_ops,
2926                                    struct trace_event_file *file,
2927                                    char *glob, char *cmd, char *param);
2928
2929 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2930                             struct hist_trigger_data *hist_data,
2931                             unsigned int n_keys)
2932 {
2933         struct hist_field *target_hist_field, *hist_field;
2934         unsigned int n, i, j;
2935
2936         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2937                 return false;
2938
2939         i = hist_data->n_vals;
2940         j = target_hist_data->n_vals;
2941
2942         for (n = 0; n < n_keys; n++) {
2943                 hist_field = hist_data->fields[i + n];
2944                 target_hist_field = target_hist_data->fields[j + n];
2945
2946                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2947                         return false;
2948                 if (hist_field->size != target_hist_field->size)
2949                         return false;
2950                 if (hist_field->is_signed != target_hist_field->is_signed)
2951                         return false;
2952         }
2953
2954         return true;
2955 }
2956
2957 static struct hist_trigger_data *
2958 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2959                      struct trace_event_file *file)
2960 {
2961         struct hist_trigger_data *hist_data;
2962         struct event_trigger_data *test;
2963         unsigned int n_keys;
2964
2965         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2966
2967         list_for_each_entry_rcu(test, &file->triggers, list) {
2968                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2969                         hist_data = test->private_data;
2970
2971                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2972                                 return hist_data;
2973                 }
2974         }
2975
2976         return NULL;
2977 }
2978
2979 static struct trace_event_file *event_file(struct trace_array *tr,
2980                                            char *system, char *event_name)
2981 {
2982         struct trace_event_file *file;
2983
2984         file = __find_event_file(tr, system, event_name);
2985         if (!file)
2986                 return ERR_PTR(-EINVAL);
2987
2988         return file;
2989 }
2990
2991 static struct hist_field *
2992 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2993                          char *system, char *event_name, char *field_name)
2994 {
2995         struct hist_field *event_var;
2996         char *synthetic_name;
2997
2998         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2999         if (!synthetic_name)
3000                 return ERR_PTR(-ENOMEM);
3001
3002         strcpy(synthetic_name, "synthetic_");
3003         strcat(synthetic_name, field_name);
3004
3005         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3006
3007         kfree(synthetic_name);
3008
3009         return event_var;
3010 }
3011
3012 /**
3013  * create_field_var_hist - Automatically create a histogram and var for a field
3014  * @target_hist_data: The target hist trigger
3015  * @subsys_name: Optional subsystem name
3016  * @event_name: Optional event name
3017  * @field_name: The name of the field (and the resulting variable)
3018  *
3019  * Hist trigger actions fetch data from variables, not directly from
3020  * events.  However, for convenience, users are allowed to directly
3021  * specify an event field in an action, which will be automatically
3022  * converted into a variable on their behalf.
3023
3024  * If a user specifies a field on an event that isn't the event the
3025  * histogram currently being defined (the target event histogram), the
3026  * only way that can be accomplished is if a new hist trigger is
3027  * created and the field variable defined on that.
3028  *
3029  * This function creates a new histogram compatible with the target
3030  * event (meaning a histogram with the same key as the target
3031  * histogram), and creates a variable for the specified field, but
3032  * with 'synthetic_' prepended to the variable name in order to avoid
3033  * collision with normal field variables.
3034  *
3035  * Return: The variable created for the field.
3036  */
3037 static struct hist_field *
3038 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3039                       char *subsys_name, char *event_name, char *field_name)
3040 {
3041         struct trace_array *tr = target_hist_data->event_file->tr;
3042         struct hist_field *event_var = ERR_PTR(-EINVAL);
3043         struct hist_trigger_data *hist_data;
3044         unsigned int i, n, first = true;
3045         struct field_var_hist *var_hist;
3046         struct trace_event_file *file;
3047         struct hist_field *key_field;
3048         char *saved_filter;
3049         char *cmd;
3050         int ret;
3051
3052         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3053                 hist_err_event("onmatch: Too many field variables defined: ",
3054                                subsys_name, event_name, field_name);
3055                 return ERR_PTR(-EINVAL);
3056         }
3057
3058         file = event_file(tr, subsys_name, event_name);
3059
3060         if (IS_ERR(file)) {
3061                 hist_err_event("onmatch: Event file not found: ",
3062                                subsys_name, event_name, field_name);
3063                 ret = PTR_ERR(file);
3064                 return ERR_PTR(ret);
3065         }
3066
3067         /*
3068          * Look for a histogram compatible with target.  We'll use the
3069          * found histogram specification to create a new matching
3070          * histogram with our variable on it.  target_hist_data is not
3071          * yet a registered histogram so we can't use that.
3072          */
3073         hist_data = find_compatible_hist(target_hist_data, file);
3074         if (!hist_data) {
3075                 hist_err_event("onmatch: Matching event histogram not found: ",
3076                                subsys_name, event_name, field_name);
3077                 return ERR_PTR(-EINVAL);
3078         }
3079
3080         /* See if a synthetic field variable has already been created */
3081         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3082                                              event_name, field_name);
3083         if (!IS_ERR_OR_NULL(event_var))
3084                 return event_var;
3085
3086         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3087         if (!var_hist)
3088                 return ERR_PTR(-ENOMEM);
3089
3090         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3091         if (!cmd) {
3092                 kfree(var_hist);
3093                 return ERR_PTR(-ENOMEM);
3094         }
3095
3096         /* Use the same keys as the compatible histogram */
3097         strcat(cmd, "keys=");
3098
3099         for_each_hist_key_field(i, hist_data) {
3100                 key_field = hist_data->fields[i];
3101                 if (!first)
3102                         strcat(cmd, ",");
3103                 strcat(cmd, key_field->field->name);
3104                 first = false;
3105         }
3106
3107         /* Create the synthetic field variable specification */
3108         strcat(cmd, ":synthetic_");
3109         strcat(cmd, field_name);
3110         strcat(cmd, "=");
3111         strcat(cmd, field_name);
3112
3113         /* Use the same filter as the compatible histogram */
3114         saved_filter = find_trigger_filter(hist_data, file);
3115         if (saved_filter) {
3116                 strcat(cmd, " if ");
3117                 strcat(cmd, saved_filter);
3118         }
3119
3120         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3121         if (!var_hist->cmd) {
3122                 kfree(cmd);
3123                 kfree(var_hist);
3124                 return ERR_PTR(-ENOMEM);
3125         }
3126
3127         /* Save the compatible histogram information */
3128         var_hist->hist_data = hist_data;
3129
3130         /* Create the new histogram with our variable */
3131         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3132                                       "", "hist", cmd);
3133         if (ret) {
3134                 kfree(cmd);
3135                 kfree(var_hist->cmd);
3136                 kfree(var_hist);
3137                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3138                                subsys_name, event_name, field_name);
3139                 return ERR_PTR(ret);
3140         }
3141
3142         kfree(cmd);
3143
3144         /* If we can't find the variable, something went wrong */
3145         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3146                                              event_name, field_name);
3147         if (IS_ERR_OR_NULL(event_var)) {
3148                 kfree(var_hist->cmd);
3149                 kfree(var_hist);
3150                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3151                                subsys_name, event_name, field_name);
3152                 return ERR_PTR(-EINVAL);
3153         }
3154
3155         n = target_hist_data->n_field_var_hists;
3156         target_hist_data->field_var_hists[n] = var_hist;
3157         target_hist_data->n_field_var_hists++;
3158
3159         return event_var;
3160 }
3161
3162 static struct hist_field *
3163 find_target_event_var(struct hist_trigger_data *hist_data,
3164                       char *subsys_name, char *event_name, char *var_name)
3165 {
3166         struct trace_event_file *file = hist_data->event_file;
3167         struct hist_field *hist_field = NULL;
3168
3169         if (subsys_name) {
3170                 struct trace_event_call *call;
3171
3172                 if (!event_name)
3173                         return NULL;
3174
3175                 call = file->event_call;
3176
3177                 if (strcmp(subsys_name, call->class->system) != 0)
3178                         return NULL;
3179
3180                 if (strcmp(event_name, trace_event_name(call)) != 0)
3181                         return NULL;
3182         }
3183
3184         hist_field = find_var_field(hist_data, var_name);
3185
3186         return hist_field;
3187 }
3188
3189 static inline void __update_field_vars(struct tracing_map_elt *elt,
3190                                        struct ring_buffer_event *rbe,
3191                                        void *rec,
3192                                        struct field_var **field_vars,
3193                                        unsigned int n_field_vars,
3194                                        unsigned int field_var_str_start)
3195 {
3196         struct hist_elt_data *elt_data = elt->private_data;
3197         unsigned int i, j, var_idx;
3198         u64 var_val;
3199
3200         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3201                 struct field_var *field_var = field_vars[i];
3202                 struct hist_field *var = field_var->var;
3203                 struct hist_field *val = field_var->val;
3204
3205                 var_val = val->fn(val, elt, rbe, rec);
3206                 var_idx = var->var.idx;
3207
3208                 if (val->flags & HIST_FIELD_FL_STRING) {
3209                         char *str = elt_data->field_var_str[j++];
3210                         char *val_str = (char *)(uintptr_t)var_val;
3211
3212                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3213                         var_val = (u64)(uintptr_t)str;
3214                 }
3215                 tracing_map_set_var(elt, var_idx, var_val);
3216         }
3217 }
3218
3219 static void update_field_vars(struct hist_trigger_data *hist_data,
3220                               struct tracing_map_elt *elt,
3221                               struct ring_buffer_event *rbe,
3222                               void *rec)
3223 {
3224         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3225                             hist_data->n_field_vars, 0);
3226 }
3227
3228 static void update_max_vars(struct hist_trigger_data *hist_data,
3229                             struct tracing_map_elt *elt,
3230                             struct ring_buffer_event *rbe,
3231                             void *rec)
3232 {
3233         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3234                             hist_data->n_max_vars, hist_data->n_field_var_str);
3235 }
3236
3237 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3238                                      struct trace_event_file *file,
3239                                      char *name, int size, const char *type)
3240 {
3241         struct hist_field *var;
3242         int idx;
3243
3244         if (find_var(hist_data, file, name) && !hist_data->remove) {
3245                 var = ERR_PTR(-EINVAL);
3246                 goto out;
3247         }
3248
3249         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3250         if (!var) {
3251                 var = ERR_PTR(-ENOMEM);
3252                 goto out;
3253         }
3254
3255         idx = tracing_map_add_var(hist_data->map);
3256         if (idx < 0) {
3257                 kfree(var);
3258                 var = ERR_PTR(-EINVAL);
3259                 goto out;
3260         }
3261
3262         var->flags = HIST_FIELD_FL_VAR;
3263         var->var.idx = idx;
3264         var->var.hist_data = var->hist_data = hist_data;
3265         var->size = size;
3266         var->var.name = kstrdup(name, GFP_KERNEL);
3267         var->type = kstrdup(type, GFP_KERNEL);
3268         if (!var->var.name || !var->type) {
3269                 kfree(var->var.name);
3270                 kfree(var->type);
3271                 kfree(var);
3272                 var = ERR_PTR(-ENOMEM);
3273         }
3274  out:
3275         return var;
3276 }
3277
3278 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3279                                           struct trace_event_file *file,
3280                                           char *field_name)
3281 {
3282         struct hist_field *val = NULL, *var = NULL;
3283         unsigned long flags = HIST_FIELD_FL_VAR;
3284         struct field_var *field_var;
3285         int ret = 0;
3286
3287         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3288                 hist_err("Too many field variables defined: ", field_name);
3289                 ret = -EINVAL;
3290                 goto err;
3291         }
3292
3293         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3294         if (IS_ERR(val)) {
3295                 hist_err("Couldn't parse field variable: ", field_name);
3296                 ret = PTR_ERR(val);
3297                 goto err;
3298         }
3299
3300         var = create_var(hist_data, file, field_name, val->size, val->type);
3301         if (IS_ERR(var)) {
3302                 hist_err("Couldn't create or find variable: ", field_name);
3303                 kfree(val);
3304                 ret = PTR_ERR(var);
3305                 goto err;
3306         }
3307
3308         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3309         if (!field_var) {
3310                 kfree(val);
3311                 kfree(var);
3312                 ret =  -ENOMEM;
3313                 goto err;
3314         }
3315
3316         field_var->var = var;
3317         field_var->val = val;
3318  out:
3319         return field_var;
3320  err:
3321         field_var = ERR_PTR(ret);
3322         goto out;
3323 }
3324
3325 /**
3326  * create_target_field_var - Automatically create a variable for a field
3327  * @target_hist_data: The target hist trigger
3328  * @s