Merge tag 'trace-v5.1' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux...
[muen/linux.git] / kernel / trace / trace_events_hist.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/mutex.h>
11 #include <linux/slab.h>
12 #include <linux/stacktrace.h>
13 #include <linux/rculist.h>
14 #include <linux/tracefs.h>
15
16 #include "tracing_map.h"
17 #include "trace.h"
18 #include "trace_dynevent.h"
19
20 #define SYNTH_SYSTEM            "synthetic"
21 #define SYNTH_FIELDS_MAX        16
22
23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
24
25 struct hist_field;
26
27 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
28                                 struct tracing_map_elt *elt,
29                                 struct ring_buffer_event *rbe,
30                                 void *event);
31
32 #define HIST_FIELD_OPERANDS_MAX 2
33 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
34 #define HIST_ACTIONS_MAX        8
35
36 enum field_op_id {
37         FIELD_OP_NONE,
38         FIELD_OP_PLUS,
39         FIELD_OP_MINUS,
40         FIELD_OP_UNARY_MINUS,
41 };
42
43 /*
44  * A hist_var (histogram variable) contains variable information for
45  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
46  * flag set.  A hist_var has a variable name e.g. ts0, and is
47  * associated with a given histogram trigger, as specified by
48  * hist_data.  The hist_var idx is the unique index assigned to the
49  * variable by the hist trigger's tracing_map.  The idx is what is
50  * used to set a variable's value and, by a variable reference, to
51  * retrieve it.
52  */
53 struct hist_var {
54         char                            *name;
55         struct hist_trigger_data        *hist_data;
56         unsigned int                    idx;
57 };
58
59 struct hist_field {
60         struct ftrace_event_field       *field;
61         unsigned long                   flags;
62         hist_field_fn_t                 fn;
63         unsigned int                    size;
64         unsigned int                    offset;
65         unsigned int                    is_signed;
66         const char                      *type;
67         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
68         struct hist_trigger_data        *hist_data;
69
70         /*
71          * Variable fields contain variable-specific info in var.
72          */
73         struct hist_var                 var;
74         enum field_op_id                operator;
75         char                            *system;
76         char                            *event_name;
77
78         /*
79          * The name field is used for EXPR and VAR_REF fields.  VAR
80          * fields contain the variable name in var.name.
81          */
82         char                            *name;
83
84         /*
85          * When a histogram trigger is hit, if it has any references
86          * to variables, the values of those variables are collected
87          * into a var_ref_vals array by resolve_var_refs().  The
88          * current value of each variable is read from the tracing_map
89          * using the hist field's hist_var.idx and entered into the
90          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
91          */
92         unsigned int                    var_ref_idx;
93         bool                            read_once;
94 };
95
96 static u64 hist_field_none(struct hist_field *field,
97                            struct tracing_map_elt *elt,
98                            struct ring_buffer_event *rbe,
99                            void *event)
100 {
101         return 0;
102 }
103
104 static u64 hist_field_counter(struct hist_field *field,
105                               struct tracing_map_elt *elt,
106                               struct ring_buffer_event *rbe,
107                               void *event)
108 {
109         return 1;
110 }
111
112 static u64 hist_field_string(struct hist_field *hist_field,
113                              struct tracing_map_elt *elt,
114                              struct ring_buffer_event *rbe,
115                              void *event)
116 {
117         char *addr = (char *)(event + hist_field->field->offset);
118
119         return (u64)(unsigned long)addr;
120 }
121
122 static u64 hist_field_dynstring(struct hist_field *hist_field,
123                                 struct tracing_map_elt *elt,
124                                 struct ring_buffer_event *rbe,
125                                 void *event)
126 {
127         u32 str_item = *(u32 *)(event + hist_field->field->offset);
128         int str_loc = str_item & 0xffff;
129         char *addr = (char *)(event + str_loc);
130
131         return (u64)(unsigned long)addr;
132 }
133
134 static u64 hist_field_pstring(struct hist_field *hist_field,
135                               struct tracing_map_elt *elt,
136                               struct ring_buffer_event *rbe,
137                               void *event)
138 {
139         char **addr = (char **)(event + hist_field->field->offset);
140
141         return (u64)(unsigned long)*addr;
142 }
143
144 static u64 hist_field_log2(struct hist_field *hist_field,
145                            struct tracing_map_elt *elt,
146                            struct ring_buffer_event *rbe,
147                            void *event)
148 {
149         struct hist_field *operand = hist_field->operands[0];
150
151         u64 val = operand->fn(operand, elt, rbe, event);
152
153         return (u64) ilog2(roundup_pow_of_two(val));
154 }
155
156 static u64 hist_field_plus(struct hist_field *hist_field,
157                            struct tracing_map_elt *elt,
158                            struct ring_buffer_event *rbe,
159                            void *event)
160 {
161         struct hist_field *operand1 = hist_field->operands[0];
162         struct hist_field *operand2 = hist_field->operands[1];
163
164         u64 val1 = operand1->fn(operand1, elt, rbe, event);
165         u64 val2 = operand2->fn(operand2, elt, rbe, event);
166
167         return val1 + val2;
168 }
169
170 static u64 hist_field_minus(struct hist_field *hist_field,
171                             struct tracing_map_elt *elt,
172                             struct ring_buffer_event *rbe,
173                             void *event)
174 {
175         struct hist_field *operand1 = hist_field->operands[0];
176         struct hist_field *operand2 = hist_field->operands[1];
177
178         u64 val1 = operand1->fn(operand1, elt, rbe, event);
179         u64 val2 = operand2->fn(operand2, elt, rbe, event);
180
181         return val1 - val2;
182 }
183
184 static u64 hist_field_unary_minus(struct hist_field *hist_field,
185                                   struct tracing_map_elt *elt,
186                                   struct ring_buffer_event *rbe,
187                                   void *event)
188 {
189         struct hist_field *operand = hist_field->operands[0];
190
191         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
192         u64 val = (u64)-sval;
193
194         return val;
195 }
196
197 #define DEFINE_HIST_FIELD_FN(type)                                      \
198         static u64 hist_field_##type(struct hist_field *hist_field,     \
199                                      struct tracing_map_elt *elt,       \
200                                      struct ring_buffer_event *rbe,     \
201                                      void *event)                       \
202 {                                                                       \
203         type *addr = (type *)(event + hist_field->field->offset);       \
204                                                                         \
205         return (u64)(unsigned long)*addr;                               \
206 }
207
208 DEFINE_HIST_FIELD_FN(s64);
209 DEFINE_HIST_FIELD_FN(u64);
210 DEFINE_HIST_FIELD_FN(s32);
211 DEFINE_HIST_FIELD_FN(u32);
212 DEFINE_HIST_FIELD_FN(s16);
213 DEFINE_HIST_FIELD_FN(u16);
214 DEFINE_HIST_FIELD_FN(s8);
215 DEFINE_HIST_FIELD_FN(u8);
216
217 #define for_each_hist_field(i, hist_data)       \
218         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
219
220 #define for_each_hist_val_field(i, hist_data)   \
221         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
222
223 #define for_each_hist_key_field(i, hist_data)   \
224         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
225
226 #define HIST_STACKTRACE_DEPTH   16
227 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
228 #define HIST_STACKTRACE_SKIP    5
229
230 #define HITCOUNT_IDX            0
231 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
232
233 enum hist_field_flags {
234         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
235         HIST_FIELD_FL_KEY               = 1 << 1,
236         HIST_FIELD_FL_STRING            = 1 << 2,
237         HIST_FIELD_FL_HEX               = 1 << 3,
238         HIST_FIELD_FL_SYM               = 1 << 4,
239         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
240         HIST_FIELD_FL_EXECNAME          = 1 << 6,
241         HIST_FIELD_FL_SYSCALL           = 1 << 7,
242         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
243         HIST_FIELD_FL_LOG2              = 1 << 9,
244         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
245         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
246         HIST_FIELD_FL_VAR               = 1 << 12,
247         HIST_FIELD_FL_EXPR              = 1 << 13,
248         HIST_FIELD_FL_VAR_REF           = 1 << 14,
249         HIST_FIELD_FL_CPU               = 1 << 15,
250         HIST_FIELD_FL_ALIAS             = 1 << 16,
251 };
252
253 struct var_defs {
254         unsigned int    n_vars;
255         char            *name[TRACING_MAP_VARS_MAX];
256         char            *expr[TRACING_MAP_VARS_MAX];
257 };
258
259 struct hist_trigger_attrs {
260         char            *keys_str;
261         char            *vals_str;
262         char            *sort_key_str;
263         char            *name;
264         char            *clock;
265         bool            pause;
266         bool            cont;
267         bool            clear;
268         bool            ts_in_usecs;
269         unsigned int    map_bits;
270
271         char            *assignment_str[TRACING_MAP_VARS_MAX];
272         unsigned int    n_assignments;
273
274         char            *action_str[HIST_ACTIONS_MAX];
275         unsigned int    n_actions;
276
277         struct var_defs var_defs;
278 };
279
280 struct field_var {
281         struct hist_field       *var;
282         struct hist_field       *val;
283 };
284
285 struct field_var_hist {
286         struct hist_trigger_data        *hist_data;
287         char                            *cmd;
288 };
289
290 struct hist_trigger_data {
291         struct hist_field               *fields[HIST_FIELDS_MAX];
292         unsigned int                    n_vals;
293         unsigned int                    n_keys;
294         unsigned int                    n_fields;
295         unsigned int                    n_vars;
296         unsigned int                    key_size;
297         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
298         unsigned int                    n_sort_keys;
299         struct trace_event_file         *event_file;
300         struct hist_trigger_attrs       *attrs;
301         struct tracing_map              *map;
302         bool                            enable_timestamps;
303         bool                            remove;
304         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
305         unsigned int                    n_var_refs;
306
307         struct action_data              *actions[HIST_ACTIONS_MAX];
308         unsigned int                    n_actions;
309
310         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
311         unsigned int                    n_field_vars;
312         unsigned int                    n_field_var_str;
313         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
314         unsigned int                    n_field_var_hists;
315
316         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
317         unsigned int                    n_save_vars;
318         unsigned int                    n_save_var_str;
319 };
320
321 static int synth_event_create(int argc, const char **argv);
322 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
323 static int synth_event_release(struct dyn_event *ev);
324 static bool synth_event_is_busy(struct dyn_event *ev);
325 static bool synth_event_match(const char *system, const char *event,
326                               struct dyn_event *ev);
327
328 static struct dyn_event_operations synth_event_ops = {
329         .create = synth_event_create,
330         .show = synth_event_show,
331         .is_busy = synth_event_is_busy,
332         .free = synth_event_release,
333         .match = synth_event_match,
334 };
335
336 struct synth_field {
337         char *type;
338         char *name;
339         size_t size;
340         bool is_signed;
341         bool is_string;
342 };
343
344 struct synth_event {
345         struct dyn_event                        devent;
346         int                                     ref;
347         char                                    *name;
348         struct synth_field                      **fields;
349         unsigned int                            n_fields;
350         unsigned int                            n_u64;
351         struct trace_event_class                class;
352         struct trace_event_call                 call;
353         struct tracepoint                       *tp;
354 };
355
356 static bool is_synth_event(struct dyn_event *ev)
357 {
358         return ev->ops == &synth_event_ops;
359 }
360
361 static struct synth_event *to_synth_event(struct dyn_event *ev)
362 {
363         return container_of(ev, struct synth_event, devent);
364 }
365
366 static bool synth_event_is_busy(struct dyn_event *ev)
367 {
368         struct synth_event *event = to_synth_event(ev);
369
370         return event->ref != 0;
371 }
372
373 static bool synth_event_match(const char *system, const char *event,
374                               struct dyn_event *ev)
375 {
376         struct synth_event *sev = to_synth_event(ev);
377
378         return strcmp(sev->name, event) == 0 &&
379                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
380 }
381
382 struct action_data;
383
384 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
385                              struct tracing_map_elt *elt, void *rec,
386                              struct ring_buffer_event *rbe, void *key,
387                              struct action_data *data, u64 *var_ref_vals);
388
389 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
390
391 enum handler_id {
392         HANDLER_ONMATCH = 1,
393         HANDLER_ONMAX,
394         HANDLER_ONCHANGE,
395 };
396
397 enum action_id {
398         ACTION_SAVE = 1,
399         ACTION_TRACE,
400         ACTION_SNAPSHOT,
401 };
402
403 struct action_data {
404         enum handler_id         handler;
405         enum action_id          action;
406         char                    *action_name;
407         action_fn_t             fn;
408
409         unsigned int            n_params;
410         char                    *params[SYNTH_FIELDS_MAX];
411
412         /*
413          * When a histogram trigger is hit, the values of any
414          * references to variables, including variables being passed
415          * as parameters to synthetic events, are collected into a
416          * var_ref_vals array.  This var_ref_idx is the index of the
417          * first param in the array to be passed to the synthetic
418          * event invocation.
419          */
420         unsigned int            var_ref_idx;
421         struct synth_event      *synth_event;
422         bool                    use_trace_keyword;
423         char                    *synth_event_name;
424
425         union {
426                 struct {
427                         char                    *event;
428                         char                    *event_system;
429                 } match_data;
430
431                 struct {
432                         /*
433                          * var_str contains the $-unstripped variable
434                          * name referenced by var_ref, and used when
435                          * printing the action.  Because var_ref
436                          * creation is deferred to create_actions(),
437                          * we need a per-action way to save it until
438                          * then, thus var_str.
439                          */
440                         char                    *var_str;
441
442                         /*
443                          * var_ref refers to the variable being
444                          * tracked e.g onmax($var).
445                          */
446                         struct hist_field       *var_ref;
447
448                         /*
449                          * track_var contains the 'invisible' tracking
450                          * variable created to keep the current
451                          * e.g. max value.
452                          */
453                         struct hist_field       *track_var;
454
455                         check_track_val_fn_t    check_val;
456                         action_fn_t             save_data;
457                 } track_data;
458         };
459 };
460
461 struct track_data {
462         u64                             track_val;
463         bool                            updated;
464
465         unsigned int                    key_len;
466         void                            *key;
467         struct tracing_map_elt          elt;
468
469         struct action_data              *action_data;
470         struct hist_trigger_data        *hist_data;
471 };
472
473 struct hist_elt_data {
474         char *comm;
475         u64 *var_ref_vals;
476         char *field_var_str[SYNTH_FIELDS_MAX];
477 };
478
479 struct snapshot_context {
480         struct tracing_map_elt  *elt;
481         void                    *key;
482 };
483
484 static void track_data_free(struct track_data *track_data)
485 {
486         struct hist_elt_data *elt_data;
487
488         if (!track_data)
489                 return;
490
491         kfree(track_data->key);
492
493         elt_data = track_data->elt.private_data;
494         if (elt_data) {
495                 kfree(elt_data->comm);
496                 kfree(elt_data);
497         }
498
499         kfree(track_data);
500 }
501
502 static struct track_data *track_data_alloc(unsigned int key_len,
503                                            struct action_data *action_data,
504                                            struct hist_trigger_data *hist_data)
505 {
506         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
507         struct hist_elt_data *elt_data;
508
509         if (!data)
510                 return ERR_PTR(-ENOMEM);
511
512         data->key = kzalloc(key_len, GFP_KERNEL);
513         if (!data->key) {
514                 track_data_free(data);
515                 return ERR_PTR(-ENOMEM);
516         }
517
518         data->key_len = key_len;
519         data->action_data = action_data;
520         data->hist_data = hist_data;
521
522         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
523         if (!elt_data) {
524                 track_data_free(data);
525                 return ERR_PTR(-ENOMEM);
526         }
527         data->elt.private_data = elt_data;
528
529         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
530         if (!elt_data->comm) {
531                 track_data_free(data);
532                 return ERR_PTR(-ENOMEM);
533         }
534
535         return data;
536 }
537
538 static char last_hist_cmd[MAX_FILTER_STR_VAL];
539 static char hist_err_str[MAX_FILTER_STR_VAL];
540
541 static void last_cmd_set(char *str)
542 {
543         if (!str)
544                 return;
545
546         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
547 }
548
549 static void hist_err(char *str, char *var)
550 {
551         int maxlen = MAX_FILTER_STR_VAL - 1;
552
553         if (!str)
554                 return;
555
556         if (strlen(hist_err_str))
557                 return;
558
559         if (!var)
560                 var = "";
561
562         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
563                 return;
564
565         strcat(hist_err_str, str);
566         strcat(hist_err_str, var);
567 }
568
569 static void hist_err_event(char *str, char *system, char *event, char *var)
570 {
571         char err[MAX_FILTER_STR_VAL];
572
573         if (system && var)
574                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
575         else if (system)
576                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
577         else
578                 strscpy(err, var, MAX_FILTER_STR_VAL);
579
580         hist_err(str, err);
581 }
582
583 static void hist_err_clear(void)
584 {
585         hist_err_str[0] = '\0';
586 }
587
588 static bool have_hist_err(void)
589 {
590         if (strlen(hist_err_str))
591                 return true;
592
593         return false;
594 }
595
596 struct synth_trace_event {
597         struct trace_entry      ent;
598         u64                     fields[];
599 };
600
601 static int synth_event_define_fields(struct trace_event_call *call)
602 {
603         struct synth_trace_event trace;
604         int offset = offsetof(typeof(trace), fields);
605         struct synth_event *event = call->data;
606         unsigned int i, size, n_u64;
607         char *name, *type;
608         bool is_signed;
609         int ret = 0;
610
611         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
612                 size = event->fields[i]->size;
613                 is_signed = event->fields[i]->is_signed;
614                 type = event->fields[i]->type;
615                 name = event->fields[i]->name;
616                 ret = trace_define_field(call, type, name, offset, size,
617                                          is_signed, FILTER_OTHER);
618                 if (ret)
619                         break;
620
621                 if (event->fields[i]->is_string) {
622                         offset += STR_VAR_LEN_MAX;
623                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
624                 } else {
625                         offset += sizeof(u64);
626                         n_u64++;
627                 }
628         }
629
630         event->n_u64 = n_u64;
631
632         return ret;
633 }
634
635 static bool synth_field_signed(char *type)
636 {
637         if (str_has_prefix(type, "u"))
638                 return false;
639
640         return true;
641 }
642
643 static int synth_field_is_string(char *type)
644 {
645         if (strstr(type, "char[") != NULL)
646                 return true;
647
648         return false;
649 }
650
651 static int synth_field_string_size(char *type)
652 {
653         char buf[4], *end, *start;
654         unsigned int len;
655         int size, err;
656
657         start = strstr(type, "char[");
658         if (start == NULL)
659                 return -EINVAL;
660         start += sizeof("char[") - 1;
661
662         end = strchr(type, ']');
663         if (!end || end < start)
664                 return -EINVAL;
665
666         len = end - start;
667         if (len > 3)
668                 return -EINVAL;
669
670         strncpy(buf, start, len);
671         buf[len] = '\0';
672
673         err = kstrtouint(buf, 0, &size);
674         if (err)
675                 return err;
676
677         if (size > STR_VAR_LEN_MAX)
678                 return -EINVAL;
679
680         return size;
681 }
682
683 static int synth_field_size(char *type)
684 {
685         int size = 0;
686
687         if (strcmp(type, "s64") == 0)
688                 size = sizeof(s64);
689         else if (strcmp(type, "u64") == 0)
690                 size = sizeof(u64);
691         else if (strcmp(type, "s32") == 0)
692                 size = sizeof(s32);
693         else if (strcmp(type, "u32") == 0)
694                 size = sizeof(u32);
695         else if (strcmp(type, "s16") == 0)
696                 size = sizeof(s16);
697         else if (strcmp(type, "u16") == 0)
698                 size = sizeof(u16);
699         else if (strcmp(type, "s8") == 0)
700                 size = sizeof(s8);
701         else if (strcmp(type, "u8") == 0)
702                 size = sizeof(u8);
703         else if (strcmp(type, "char") == 0)
704                 size = sizeof(char);
705         else if (strcmp(type, "unsigned char") == 0)
706                 size = sizeof(unsigned char);
707         else if (strcmp(type, "int") == 0)
708                 size = sizeof(int);
709         else if (strcmp(type, "unsigned int") == 0)
710                 size = sizeof(unsigned int);
711         else if (strcmp(type, "long") == 0)
712                 size = sizeof(long);
713         else if (strcmp(type, "unsigned long") == 0)
714                 size = sizeof(unsigned long);
715         else if (strcmp(type, "pid_t") == 0)
716                 size = sizeof(pid_t);
717         else if (synth_field_is_string(type))
718                 size = synth_field_string_size(type);
719
720         return size;
721 }
722
723 static const char *synth_field_fmt(char *type)
724 {
725         const char *fmt = "%llu";
726
727         if (strcmp(type, "s64") == 0)
728                 fmt = "%lld";
729         else if (strcmp(type, "u64") == 0)
730                 fmt = "%llu";
731         else if (strcmp(type, "s32") == 0)
732                 fmt = "%d";
733         else if (strcmp(type, "u32") == 0)
734                 fmt = "%u";
735         else if (strcmp(type, "s16") == 0)
736                 fmt = "%d";
737         else if (strcmp(type, "u16") == 0)
738                 fmt = "%u";
739         else if (strcmp(type, "s8") == 0)
740                 fmt = "%d";
741         else if (strcmp(type, "u8") == 0)
742                 fmt = "%u";
743         else if (strcmp(type, "char") == 0)
744                 fmt = "%d";
745         else if (strcmp(type, "unsigned char") == 0)
746                 fmt = "%u";
747         else if (strcmp(type, "int") == 0)
748                 fmt = "%d";
749         else if (strcmp(type, "unsigned int") == 0)
750                 fmt = "%u";
751         else if (strcmp(type, "long") == 0)
752                 fmt = "%ld";
753         else if (strcmp(type, "unsigned long") == 0)
754                 fmt = "%lu";
755         else if (strcmp(type, "pid_t") == 0)
756                 fmt = "%d";
757         else if (synth_field_is_string(type))
758                 fmt = "%s";
759
760         return fmt;
761 }
762
763 static enum print_line_t print_synth_event(struct trace_iterator *iter,
764                                            int flags,
765                                            struct trace_event *event)
766 {
767         struct trace_array *tr = iter->tr;
768         struct trace_seq *s = &iter->seq;
769         struct synth_trace_event *entry;
770         struct synth_event *se;
771         unsigned int i, n_u64;
772         char print_fmt[32];
773         const char *fmt;
774
775         entry = (struct synth_trace_event *)iter->ent;
776         se = container_of(event, struct synth_event, call.event);
777
778         trace_seq_printf(s, "%s: ", se->name);
779
780         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
781                 if (trace_seq_has_overflowed(s))
782                         goto end;
783
784                 fmt = synth_field_fmt(se->fields[i]->type);
785
786                 /* parameter types */
787                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
788                         trace_seq_printf(s, "%s ", fmt);
789
790                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
791
792                 /* parameter values */
793                 if (se->fields[i]->is_string) {
794                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
795                                          (char *)&entry->fields[n_u64],
796                                          i == se->n_fields - 1 ? "" : " ");
797                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
798                 } else {
799                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
800                                          entry->fields[n_u64],
801                                          i == se->n_fields - 1 ? "" : " ");
802                         n_u64++;
803                 }
804         }
805 end:
806         trace_seq_putc(s, '\n');
807
808         return trace_handle_return(s);
809 }
810
811 static struct trace_event_functions synth_event_funcs = {
812         .trace          = print_synth_event
813 };
814
815 static notrace void trace_event_raw_event_synth(void *__data,
816                                                 u64 *var_ref_vals,
817                                                 unsigned int var_ref_idx)
818 {
819         struct trace_event_file *trace_file = __data;
820         struct synth_trace_event *entry;
821         struct trace_event_buffer fbuffer;
822         struct ring_buffer *buffer;
823         struct synth_event *event;
824         unsigned int i, n_u64;
825         int fields_size = 0;
826
827         event = trace_file->event_call->data;
828
829         if (trace_trigger_soft_disabled(trace_file))
830                 return;
831
832         fields_size = event->n_u64 * sizeof(u64);
833
834         /*
835          * Avoid ring buffer recursion detection, as this event
836          * is being performed within another event.
837          */
838         buffer = trace_file->tr->trace_buffer.buffer;
839         ring_buffer_nest_start(buffer);
840
841         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
842                                            sizeof(*entry) + fields_size);
843         if (!entry)
844                 goto out;
845
846         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
847                 if (event->fields[i]->is_string) {
848                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
849                         char *str_field = (char *)&entry->fields[n_u64];
850
851                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
852                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
853                 } else {
854                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
855                         n_u64++;
856                 }
857         }
858
859         trace_event_buffer_commit(&fbuffer);
860 out:
861         ring_buffer_nest_end(buffer);
862 }
863
864 static void free_synth_event_print_fmt(struct trace_event_call *call)
865 {
866         if (call) {
867                 kfree(call->print_fmt);
868                 call->print_fmt = NULL;
869         }
870 }
871
872 static int __set_synth_event_print_fmt(struct synth_event *event,
873                                        char *buf, int len)
874 {
875         const char *fmt;
876         int pos = 0;
877         int i;
878
879         /* When len=0, we just calculate the needed length */
880 #define LEN_OR_ZERO (len ? len - pos : 0)
881
882         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
883         for (i = 0; i < event->n_fields; i++) {
884                 fmt = synth_field_fmt(event->fields[i]->type);
885                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
886                                 event->fields[i]->name, fmt,
887                                 i == event->n_fields - 1 ? "" : ", ");
888         }
889         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
890
891         for (i = 0; i < event->n_fields; i++) {
892                 pos += snprintf(buf + pos, LEN_OR_ZERO,
893                                 ", REC->%s", event->fields[i]->name);
894         }
895
896 #undef LEN_OR_ZERO
897
898         /* return the length of print_fmt */
899         return pos;
900 }
901
902 static int set_synth_event_print_fmt(struct trace_event_call *call)
903 {
904         struct synth_event *event = call->data;
905         char *print_fmt;
906         int len;
907
908         /* First: called with 0 length to calculate the needed length */
909         len = __set_synth_event_print_fmt(event, NULL, 0);
910
911         print_fmt = kmalloc(len + 1, GFP_KERNEL);
912         if (!print_fmt)
913                 return -ENOMEM;
914
915         /* Second: actually write the @print_fmt */
916         __set_synth_event_print_fmt(event, print_fmt, len + 1);
917         call->print_fmt = print_fmt;
918
919         return 0;
920 }
921
922 static void free_synth_field(struct synth_field *field)
923 {
924         kfree(field->type);
925         kfree(field->name);
926         kfree(field);
927 }
928
929 static struct synth_field *parse_synth_field(int argc, const char **argv,
930                                              int *consumed)
931 {
932         struct synth_field *field;
933         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
934         int len, ret = 0;
935
936         if (field_type[0] == ';')
937                 field_type++;
938
939         if (!strcmp(field_type, "unsigned")) {
940                 if (argc < 3)
941                         return ERR_PTR(-EINVAL);
942                 prefix = "unsigned ";
943                 field_type = argv[1];
944                 field_name = argv[2];
945                 *consumed = 3;
946         } else {
947                 field_name = argv[1];
948                 *consumed = 2;
949         }
950
951         field = kzalloc(sizeof(*field), GFP_KERNEL);
952         if (!field)
953                 return ERR_PTR(-ENOMEM);
954
955         len = strlen(field_name);
956         array = strchr(field_name, '[');
957         if (array)
958                 len -= strlen(array);
959         else if (field_name[len - 1] == ';')
960                 len--;
961
962         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
963         if (!field->name) {
964                 ret = -ENOMEM;
965                 goto free;
966         }
967
968         if (field_type[0] == ';')
969                 field_type++;
970         len = strlen(field_type) + 1;
971         if (array)
972                 len += strlen(array);
973         if (prefix)
974                 len += strlen(prefix);
975
976         field->type = kzalloc(len, GFP_KERNEL);
977         if (!field->type) {
978                 ret = -ENOMEM;
979                 goto free;
980         }
981         if (prefix)
982                 strcat(field->type, prefix);
983         strcat(field->type, field_type);
984         if (array) {
985                 strcat(field->type, array);
986                 if (field->type[len - 1] == ';')
987                         field->type[len - 1] = '\0';
988         }
989
990         field->size = synth_field_size(field->type);
991         if (!field->size) {
992                 ret = -EINVAL;
993                 goto free;
994         }
995
996         if (synth_field_is_string(field->type))
997                 field->is_string = true;
998
999         field->is_signed = synth_field_signed(field->type);
1000
1001  out:
1002         return field;
1003  free:
1004         free_synth_field(field);
1005         field = ERR_PTR(ret);
1006         goto out;
1007 }
1008
1009 static void free_synth_tracepoint(struct tracepoint *tp)
1010 {
1011         if (!tp)
1012                 return;
1013
1014         kfree(tp->name);
1015         kfree(tp);
1016 }
1017
1018 static struct tracepoint *alloc_synth_tracepoint(char *name)
1019 {
1020         struct tracepoint *tp;
1021
1022         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1023         if (!tp)
1024                 return ERR_PTR(-ENOMEM);
1025
1026         tp->name = kstrdup(name, GFP_KERNEL);
1027         if (!tp->name) {
1028                 kfree(tp);
1029                 return ERR_PTR(-ENOMEM);
1030         }
1031
1032         return tp;
1033 }
1034
1035 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1036                                     unsigned int var_ref_idx);
1037
1038 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1039                                unsigned int var_ref_idx)
1040 {
1041         struct tracepoint *tp = event->tp;
1042
1043         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1044                 struct tracepoint_func *probe_func_ptr;
1045                 synth_probe_func_t probe_func;
1046                 void *__data;
1047
1048                 if (!(cpu_online(raw_smp_processor_id())))
1049                         return;
1050
1051                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1052                 if (probe_func_ptr) {
1053                         do {
1054                                 probe_func = probe_func_ptr->func;
1055                                 __data = probe_func_ptr->data;
1056                                 probe_func(__data, var_ref_vals, var_ref_idx);
1057                         } while ((++probe_func_ptr)->func);
1058                 }
1059         }
1060 }
1061
1062 static struct synth_event *find_synth_event(const char *name)
1063 {
1064         struct dyn_event *pos;
1065         struct synth_event *event;
1066
1067         for_each_dyn_event(pos) {
1068                 if (!is_synth_event(pos))
1069                         continue;
1070                 event = to_synth_event(pos);
1071                 if (strcmp(event->name, name) == 0)
1072                         return event;
1073         }
1074
1075         return NULL;
1076 }
1077
1078 static int register_synth_event(struct synth_event *event)
1079 {
1080         struct trace_event_call *call = &event->call;
1081         int ret = 0;
1082
1083         event->call.class = &event->class;
1084         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1085         if (!event->class.system) {
1086                 ret = -ENOMEM;
1087                 goto out;
1088         }
1089
1090         event->tp = alloc_synth_tracepoint(event->name);
1091         if (IS_ERR(event->tp)) {
1092                 ret = PTR_ERR(event->tp);
1093                 event->tp = NULL;
1094                 goto out;
1095         }
1096
1097         INIT_LIST_HEAD(&call->class->fields);
1098         call->event.funcs = &synth_event_funcs;
1099         call->class->define_fields = synth_event_define_fields;
1100
1101         ret = register_trace_event(&call->event);
1102         if (!ret) {
1103                 ret = -ENODEV;
1104                 goto out;
1105         }
1106         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1107         call->class->reg = trace_event_reg;
1108         call->class->probe = trace_event_raw_event_synth;
1109         call->data = event;
1110         call->tp = event->tp;
1111
1112         ret = trace_add_event_call(call);
1113         if (ret) {
1114                 pr_warn("Failed to register synthetic event: %s\n",
1115                         trace_event_name(call));
1116                 goto err;
1117         }
1118
1119         ret = set_synth_event_print_fmt(call);
1120         if (ret < 0) {
1121                 trace_remove_event_call(call);
1122                 goto err;
1123         }
1124  out:
1125         return ret;
1126  err:
1127         unregister_trace_event(&call->event);
1128         goto out;
1129 }
1130
1131 static int unregister_synth_event(struct synth_event *event)
1132 {
1133         struct trace_event_call *call = &event->call;
1134         int ret;
1135
1136         ret = trace_remove_event_call(call);
1137
1138         return ret;
1139 }
1140
1141 static void free_synth_event(struct synth_event *event)
1142 {
1143         unsigned int i;
1144
1145         if (!event)
1146                 return;
1147
1148         for (i = 0; i < event->n_fields; i++)
1149                 free_synth_field(event->fields[i]);
1150
1151         kfree(event->fields);
1152         kfree(event->name);
1153         kfree(event->class.system);
1154         free_synth_tracepoint(event->tp);
1155         free_synth_event_print_fmt(&event->call);
1156         kfree(event);
1157 }
1158
1159 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1160                                              struct synth_field **fields)
1161 {
1162         struct synth_event *event;
1163         unsigned int i;
1164
1165         event = kzalloc(sizeof(*event), GFP_KERNEL);
1166         if (!event) {
1167                 event = ERR_PTR(-ENOMEM);
1168                 goto out;
1169         }
1170
1171         event->name = kstrdup(name, GFP_KERNEL);
1172         if (!event->name) {
1173                 kfree(event);
1174                 event = ERR_PTR(-ENOMEM);
1175                 goto out;
1176         }
1177
1178         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1179         if (!event->fields) {
1180                 free_synth_event(event);
1181                 event = ERR_PTR(-ENOMEM);
1182                 goto out;
1183         }
1184
1185         dyn_event_init(&event->devent, &synth_event_ops);
1186
1187         for (i = 0; i < n_fields; i++)
1188                 event->fields[i] = fields[i];
1189
1190         event->n_fields = n_fields;
1191  out:
1192         return event;
1193 }
1194
1195 static void action_trace(struct hist_trigger_data *hist_data,
1196                          struct tracing_map_elt *elt, void *rec,
1197                          struct ring_buffer_event *rbe, void *key,
1198                          struct action_data *data, u64 *var_ref_vals)
1199 {
1200         struct synth_event *event = data->synth_event;
1201
1202         trace_synth(event, var_ref_vals, data->var_ref_idx);
1203 }
1204
1205 struct hist_var_data {
1206         struct list_head list;
1207         struct hist_trigger_data *hist_data;
1208 };
1209
1210 static int __create_synth_event(int argc, const char *name, const char **argv)
1211 {
1212         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1213         struct synth_event *event = NULL;
1214         int i, consumed = 0, n_fields = 0, ret = 0;
1215
1216         /*
1217          * Argument syntax:
1218          *  - Add synthetic event: <event_name> field[;field] ...
1219          *  - Remove synthetic event: !<event_name> field[;field] ...
1220          *      where 'field' = type field_name
1221          */
1222
1223         if (name[0] == '\0' || argc < 1)
1224                 return -EINVAL;
1225
1226         mutex_lock(&event_mutex);
1227
1228         event = find_synth_event(name);
1229         if (event) {
1230                 ret = -EEXIST;
1231                 goto out;
1232         }
1233
1234         for (i = 0; i < argc - 1; i++) {
1235                 if (strcmp(argv[i], ";") == 0)
1236                         continue;
1237                 if (n_fields == SYNTH_FIELDS_MAX) {
1238                         ret = -EINVAL;
1239                         goto err;
1240                 }
1241
1242                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1243                 if (IS_ERR(field)) {
1244                         ret = PTR_ERR(field);
1245                         goto err;
1246                 }
1247                 fields[n_fields++] = field;
1248                 i += consumed - 1;
1249         }
1250
1251         if (i < argc && strcmp(argv[i], ";") != 0) {
1252                 ret = -EINVAL;
1253                 goto err;
1254         }
1255
1256         event = alloc_synth_event(name, n_fields, fields);
1257         if (IS_ERR(event)) {
1258                 ret = PTR_ERR(event);
1259                 event = NULL;
1260                 goto err;
1261         }
1262         ret = register_synth_event(event);
1263         if (!ret)
1264                 dyn_event_add(&event->devent);
1265         else
1266                 free_synth_event(event);
1267  out:
1268         mutex_unlock(&event_mutex);
1269
1270         return ret;
1271  err:
1272         for (i = 0; i < n_fields; i++)
1273                 free_synth_field(fields[i]);
1274
1275         goto out;
1276 }
1277
1278 static int create_or_delete_synth_event(int argc, char **argv)
1279 {
1280         const char *name = argv[0];
1281         struct synth_event *event = NULL;
1282         int ret;
1283
1284         /* trace_run_command() ensures argc != 0 */
1285         if (name[0] == '!') {
1286                 mutex_lock(&event_mutex);
1287                 event = find_synth_event(name + 1);
1288                 if (event) {
1289                         if (event->ref)
1290                                 ret = -EBUSY;
1291                         else {
1292                                 ret = unregister_synth_event(event);
1293                                 if (!ret) {
1294                                         dyn_event_remove(&event->devent);
1295                                         free_synth_event(event);
1296                                 }
1297                         }
1298                 } else
1299                         ret = -ENOENT;
1300                 mutex_unlock(&event_mutex);
1301                 return ret;
1302         }
1303
1304         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1305         return ret == -ECANCELED ? -EINVAL : ret;
1306 }
1307
1308 static int synth_event_create(int argc, const char **argv)
1309 {
1310         const char *name = argv[0];
1311         int len;
1312
1313         if (name[0] != 's' || name[1] != ':')
1314                 return -ECANCELED;
1315         name += 2;
1316
1317         /* This interface accepts group name prefix */
1318         if (strchr(name, '/')) {
1319                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1320                 if (len == 0)
1321                         return -EINVAL;
1322                 name += len;
1323         }
1324         return __create_synth_event(argc - 1, name, argv + 1);
1325 }
1326
1327 static int synth_event_release(struct dyn_event *ev)
1328 {
1329         struct synth_event *event = to_synth_event(ev);
1330         int ret;
1331
1332         if (event->ref)
1333                 return -EBUSY;
1334
1335         ret = unregister_synth_event(event);
1336         if (ret)
1337                 return ret;
1338
1339         dyn_event_remove(ev);
1340         free_synth_event(event);
1341         return 0;
1342 }
1343
1344 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1345 {
1346         struct synth_field *field;
1347         unsigned int i;
1348
1349         seq_printf(m, "%s\t", event->name);
1350
1351         for (i = 0; i < event->n_fields; i++) {
1352                 field = event->fields[i];
1353
1354                 /* parameter values */
1355                 seq_printf(m, "%s %s%s", field->type, field->name,
1356                            i == event->n_fields - 1 ? "" : "; ");
1357         }
1358
1359         seq_putc(m, '\n');
1360
1361         return 0;
1362 }
1363
1364 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1365 {
1366         struct synth_event *event = to_synth_event(ev);
1367
1368         seq_printf(m, "s:%s/", event->class.system);
1369
1370         return __synth_event_show(m, event);
1371 }
1372
1373 static int synth_events_seq_show(struct seq_file *m, void *v)
1374 {
1375         struct dyn_event *ev = v;
1376
1377         if (!is_synth_event(ev))
1378                 return 0;
1379
1380         return __synth_event_show(m, to_synth_event(ev));
1381 }
1382
1383 static const struct seq_operations synth_events_seq_op = {
1384         .start  = dyn_event_seq_start,
1385         .next   = dyn_event_seq_next,
1386         .stop   = dyn_event_seq_stop,
1387         .show   = synth_events_seq_show,
1388 };
1389
1390 static int synth_events_open(struct inode *inode, struct file *file)
1391 {
1392         int ret;
1393
1394         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1395                 ret = dyn_events_release_all(&synth_event_ops);
1396                 if (ret < 0)
1397                         return ret;
1398         }
1399
1400         return seq_open(file, &synth_events_seq_op);
1401 }
1402
1403 static ssize_t synth_events_write(struct file *file,
1404                                   const char __user *buffer,
1405                                   size_t count, loff_t *ppos)
1406 {
1407         return trace_parse_run_command(file, buffer, count, ppos,
1408                                        create_or_delete_synth_event);
1409 }
1410
1411 static const struct file_operations synth_events_fops = {
1412         .open           = synth_events_open,
1413         .write          = synth_events_write,
1414         .read           = seq_read,
1415         .llseek         = seq_lseek,
1416         .release        = seq_release,
1417 };
1418
1419 static u64 hist_field_timestamp(struct hist_field *hist_field,
1420                                 struct tracing_map_elt *elt,
1421                                 struct ring_buffer_event *rbe,
1422                                 void *event)
1423 {
1424         struct hist_trigger_data *hist_data = hist_field->hist_data;
1425         struct trace_array *tr = hist_data->event_file->tr;
1426
1427         u64 ts = ring_buffer_event_time_stamp(rbe);
1428
1429         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1430                 ts = ns2usecs(ts);
1431
1432         return ts;
1433 }
1434
1435 static u64 hist_field_cpu(struct hist_field *hist_field,
1436                           struct tracing_map_elt *elt,
1437                           struct ring_buffer_event *rbe,
1438                           void *event)
1439 {
1440         int cpu = smp_processor_id();
1441
1442         return cpu;
1443 }
1444
1445 /**
1446  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1447  * @hist_field: The VAR_REF field to check
1448  * @var_data: The hist trigger that owns the variable
1449  * @var_idx: The trigger variable identifier
1450  *
1451  * Check the given VAR_REF field to see whether or not it references
1452  * the given variable associated with the given trigger.
1453  *
1454  * Return: The VAR_REF field if it does reference the variable, NULL if not
1455  */
1456 static struct hist_field *
1457 check_field_for_var_ref(struct hist_field *hist_field,
1458                         struct hist_trigger_data *var_data,
1459                         unsigned int var_idx)
1460 {
1461         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1462
1463         if (hist_field && hist_field->var.idx == var_idx &&
1464             hist_field->var.hist_data == var_data)
1465                 return hist_field;
1466
1467         return NULL;
1468 }
1469
1470 /**
1471  * find_var_ref - Check if a trigger has a reference to a trigger variable
1472  * @hist_data: The hist trigger that might have a reference to the variable
1473  * @var_data: The hist trigger that owns the variable
1474  * @var_idx: The trigger variable identifier
1475  *
1476  * Check the list of var_refs[] on the first hist trigger to see
1477  * whether any of them are references to the variable on the second
1478  * trigger.
1479  *
1480  * Return: The VAR_REF field referencing the variable if so, NULL if not
1481  */
1482 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1483                                        struct hist_trigger_data *var_data,
1484                                        unsigned int var_idx)
1485 {
1486         struct hist_field *hist_field;
1487         unsigned int i;
1488
1489         for (i = 0; i < hist_data->n_var_refs; i++) {
1490                 hist_field = hist_data->var_refs[i];
1491                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1492                         return hist_field;
1493         }
1494
1495         return NULL;
1496 }
1497
1498 /**
1499  * find_any_var_ref - Check if there is a reference to a given trigger variable
1500  * @hist_data: The hist trigger
1501  * @var_idx: The trigger variable identifier
1502  *
1503  * Check to see whether the given variable is currently referenced by
1504  * any other trigger.
1505  *
1506  * The trigger the variable is defined on is explicitly excluded - the
1507  * assumption being that a self-reference doesn't prevent a trigger
1508  * from being removed.
1509  *
1510  * Return: The VAR_REF field referencing the variable if so, NULL if not
1511  */
1512 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1513                                            unsigned int var_idx)
1514 {
1515         struct trace_array *tr = hist_data->event_file->tr;
1516         struct hist_field *found = NULL;
1517         struct hist_var_data *var_data;
1518
1519         list_for_each_entry(var_data, &tr->hist_vars, list) {
1520                 if (var_data->hist_data == hist_data)
1521                         continue;
1522                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1523                 if (found)
1524                         break;
1525         }
1526
1527         return found;
1528 }
1529
1530 /**
1531  * check_var_refs - Check if there is a reference to any of trigger's variables
1532  * @hist_data: The hist trigger
1533  *
1534  * A trigger can define one or more variables.  If any one of them is
1535  * currently referenced by any other trigger, this function will
1536  * determine that.
1537
1538  * Typically used to determine whether or not a trigger can be removed
1539  * - if there are any references to a trigger's variables, it cannot.
1540  *
1541  * Return: True if there is a reference to any of trigger's variables
1542  */
1543 static bool check_var_refs(struct hist_trigger_data *hist_data)
1544 {
1545         struct hist_field *field;
1546         bool found = false;
1547         int i;
1548
1549         for_each_hist_field(i, hist_data) {
1550                 field = hist_data->fields[i];
1551                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1552                         if (find_any_var_ref(hist_data, field->var.idx)) {
1553                                 found = true;
1554                                 break;
1555                         }
1556                 }
1557         }
1558
1559         return found;
1560 }
1561
1562 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1563 {
1564         struct trace_array *tr = hist_data->event_file->tr;
1565         struct hist_var_data *var_data, *found = NULL;
1566
1567         list_for_each_entry(var_data, &tr->hist_vars, list) {
1568                 if (var_data->hist_data == hist_data) {
1569                         found = var_data;
1570                         break;
1571                 }
1572         }
1573
1574         return found;
1575 }
1576
1577 static bool field_has_hist_vars(struct hist_field *hist_field,
1578                                 unsigned int level)
1579 {
1580         int i;
1581
1582         if (level > 3)
1583                 return false;
1584
1585         if (!hist_field)
1586                 return false;
1587
1588         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1589             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1590                 return true;
1591
1592         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1593                 struct hist_field *operand;
1594
1595                 operand = hist_field->operands[i];
1596                 if (field_has_hist_vars(operand, level + 1))
1597                         return true;
1598         }
1599
1600         return false;
1601 }
1602
1603 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1604 {
1605         struct hist_field *hist_field;
1606         int i;
1607
1608         for_each_hist_field(i, hist_data) {
1609                 hist_field = hist_data->fields[i];
1610                 if (field_has_hist_vars(hist_field, 0))
1611                         return true;
1612         }
1613
1614         return false;
1615 }
1616
1617 static int save_hist_vars(struct hist_trigger_data *hist_data)
1618 {
1619         struct trace_array *tr = hist_data->event_file->tr;
1620         struct hist_var_data *var_data;
1621
1622         var_data = find_hist_vars(hist_data);
1623         if (var_data)
1624                 return 0;
1625
1626         if (trace_array_get(tr) < 0)
1627                 return -ENODEV;
1628
1629         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1630         if (!var_data) {
1631                 trace_array_put(tr);
1632                 return -ENOMEM;
1633         }
1634
1635         var_data->hist_data = hist_data;
1636         list_add(&var_data->list, &tr->hist_vars);
1637
1638         return 0;
1639 }
1640
1641 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1642 {
1643         struct trace_array *tr = hist_data->event_file->tr;
1644         struct hist_var_data *var_data;
1645
1646         var_data = find_hist_vars(hist_data);
1647         if (!var_data)
1648                 return;
1649
1650         if (WARN_ON(check_var_refs(hist_data)))
1651                 return;
1652
1653         list_del(&var_data->list);
1654
1655         kfree(var_data);
1656
1657         trace_array_put(tr);
1658 }
1659
1660 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1661                                          const char *var_name)
1662 {
1663         struct hist_field *hist_field, *found = NULL;
1664         int i;
1665
1666         for_each_hist_field(i, hist_data) {
1667                 hist_field = hist_data->fields[i];
1668                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1669                     strcmp(hist_field->var.name, var_name) == 0) {
1670                         found = hist_field;
1671                         break;
1672                 }
1673         }
1674
1675         return found;
1676 }
1677
1678 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1679                                    struct trace_event_file *file,
1680                                    const char *var_name)
1681 {
1682         struct hist_trigger_data *test_data;
1683         struct event_trigger_data *test;
1684         struct hist_field *hist_field;
1685
1686         hist_field = find_var_field(hist_data, var_name);
1687         if (hist_field)
1688                 return hist_field;
1689
1690         list_for_each_entry_rcu(test, &file->triggers, list) {
1691                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1692                         test_data = test->private_data;
1693                         hist_field = find_var_field(test_data, var_name);
1694                         if (hist_field)
1695                                 return hist_field;
1696                 }
1697         }
1698
1699         return NULL;
1700 }
1701
1702 static struct trace_event_file *find_var_file(struct trace_array *tr,
1703                                               char *system,
1704                                               char *event_name,
1705                                               char *var_name)
1706 {
1707         struct hist_trigger_data *var_hist_data;
1708         struct hist_var_data *var_data;
1709         struct trace_event_file *file, *found = NULL;
1710
1711         if (system)
1712                 return find_event_file(tr, system, event_name);
1713
1714         list_for_each_entry(var_data, &tr->hist_vars, list) {
1715                 var_hist_data = var_data->hist_data;
1716                 file = var_hist_data->event_file;
1717                 if (file == found)
1718                         continue;
1719
1720                 if (find_var_field(var_hist_data, var_name)) {
1721                         if (found) {
1722                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1723                                 return NULL;
1724                         }
1725
1726                         found = file;
1727                 }
1728         }
1729
1730         return found;
1731 }
1732
1733 static struct hist_field *find_file_var(struct trace_event_file *file,
1734                                         const char *var_name)
1735 {
1736         struct hist_trigger_data *test_data;
1737         struct event_trigger_data *test;
1738         struct hist_field *hist_field;
1739
1740         list_for_each_entry_rcu(test, &file->triggers, list) {
1741                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1742                         test_data = test->private_data;
1743                         hist_field = find_var_field(test_data, var_name);
1744                         if (hist_field)
1745                                 return hist_field;
1746                 }
1747         }
1748
1749         return NULL;
1750 }
1751
1752 static struct hist_field *
1753 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1754 {
1755         struct trace_array *tr = hist_data->event_file->tr;
1756         struct hist_field *hist_field, *found = NULL;
1757         struct trace_event_file *file;
1758         unsigned int i;
1759
1760         for (i = 0; i < hist_data->n_actions; i++) {
1761                 struct action_data *data = hist_data->actions[i];
1762
1763                 if (data->handler == HANDLER_ONMATCH) {
1764                         char *system = data->match_data.event_system;
1765                         char *event_name = data->match_data.event;
1766
1767                         file = find_var_file(tr, system, event_name, var_name);
1768                         if (!file)
1769                                 continue;
1770                         hist_field = find_file_var(file, var_name);
1771                         if (hist_field) {
1772                                 if (found) {
1773                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1774                                         return ERR_PTR(-EINVAL);
1775                                 }
1776
1777                                 found = hist_field;
1778                         }
1779                 }
1780         }
1781         return found;
1782 }
1783
1784 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1785                                          char *system,
1786                                          char *event_name,
1787                                          char *var_name)
1788 {
1789         struct trace_array *tr = hist_data->event_file->tr;
1790         struct hist_field *hist_field = NULL;
1791         struct trace_event_file *file;
1792
1793         if (!system || !event_name) {
1794                 hist_field = find_match_var(hist_data, var_name);
1795                 if (IS_ERR(hist_field))
1796                         return NULL;
1797                 if (hist_field)
1798                         return hist_field;
1799         }
1800
1801         file = find_var_file(tr, system, event_name, var_name);
1802         if (!file)
1803                 return NULL;
1804
1805         hist_field = find_file_var(file, var_name);
1806
1807         return hist_field;
1808 }
1809
1810 static u64 hist_field_var_ref(struct hist_field *hist_field,
1811                               struct tracing_map_elt *elt,
1812                               struct ring_buffer_event *rbe,
1813                               void *event)
1814 {
1815         struct hist_elt_data *elt_data;
1816         u64 var_val = 0;
1817
1818         elt_data = elt->private_data;
1819         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1820
1821         return var_val;
1822 }
1823
1824 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1825                              u64 *var_ref_vals, bool self)
1826 {
1827         struct hist_trigger_data *var_data;
1828         struct tracing_map_elt *var_elt;
1829         struct hist_field *hist_field;
1830         unsigned int i, var_idx;
1831         bool resolved = true;
1832         u64 var_val = 0;
1833
1834         for (i = 0; i < hist_data->n_var_refs; i++) {
1835                 hist_field = hist_data->var_refs[i];
1836                 var_idx = hist_field->var.idx;
1837                 var_data = hist_field->var.hist_data;
1838
1839                 if (var_data == NULL) {
1840                         resolved = false;
1841                         break;
1842                 }
1843
1844                 if ((self && var_data != hist_data) ||
1845                     (!self && var_data == hist_data))
1846                         continue;
1847
1848                 var_elt = tracing_map_lookup(var_data->map, key);
1849                 if (!var_elt) {
1850                         resolved = false;
1851                         break;
1852                 }
1853
1854                 if (!tracing_map_var_set(var_elt, var_idx)) {
1855                         resolved = false;
1856                         break;
1857                 }
1858
1859                 if (self || !hist_field->read_once)
1860                         var_val = tracing_map_read_var(var_elt, var_idx);
1861                 else
1862                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1863
1864                 var_ref_vals[i] = var_val;
1865         }
1866
1867         return resolved;
1868 }
1869
1870 static const char *hist_field_name(struct hist_field *field,
1871                                    unsigned int level)
1872 {
1873         const char *field_name = "";
1874
1875         if (level > 1)
1876                 return field_name;
1877
1878         if (field->field)
1879                 field_name = field->field->name;
1880         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1881                  field->flags & HIST_FIELD_FL_ALIAS)
1882                 field_name = hist_field_name(field->operands[0], ++level);
1883         else if (field->flags & HIST_FIELD_FL_CPU)
1884                 field_name = "cpu";
1885         else if (field->flags & HIST_FIELD_FL_EXPR ||
1886                  field->flags & HIST_FIELD_FL_VAR_REF) {
1887                 if (field->system) {
1888                         static char full_name[MAX_FILTER_STR_VAL];
1889
1890                         strcat(full_name, field->system);
1891                         strcat(full_name, ".");
1892                         strcat(full_name, field->event_name);
1893                         strcat(full_name, ".");
1894                         strcat(full_name, field->name);
1895                         field_name = full_name;
1896                 } else
1897                         field_name = field->name;
1898         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1899                 field_name = "common_timestamp";
1900
1901         if (field_name == NULL)
1902                 field_name = "";
1903
1904         return field_name;
1905 }
1906
1907 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1908 {
1909         hist_field_fn_t fn = NULL;
1910
1911         switch (field_size) {
1912         case 8:
1913                 if (field_is_signed)
1914                         fn = hist_field_s64;
1915                 else
1916                         fn = hist_field_u64;
1917                 break;
1918         case 4:
1919                 if (field_is_signed)
1920                         fn = hist_field_s32;
1921                 else
1922                         fn = hist_field_u32;
1923                 break;
1924         case 2:
1925                 if (field_is_signed)
1926                         fn = hist_field_s16;
1927                 else
1928                         fn = hist_field_u16;
1929                 break;
1930         case 1:
1931                 if (field_is_signed)
1932                         fn = hist_field_s8;
1933                 else
1934                         fn = hist_field_u8;
1935                 break;
1936         }
1937
1938         return fn;
1939 }
1940
1941 static int parse_map_size(char *str)
1942 {
1943         unsigned long size, map_bits;
1944         int ret;
1945
1946         strsep(&str, "=");
1947         if (!str) {
1948                 ret = -EINVAL;
1949                 goto out;
1950         }
1951
1952         ret = kstrtoul(str, 0, &size);
1953         if (ret)
1954                 goto out;
1955
1956         map_bits = ilog2(roundup_pow_of_two(size));
1957         if (map_bits < TRACING_MAP_BITS_MIN ||
1958             map_bits > TRACING_MAP_BITS_MAX)
1959                 ret = -EINVAL;
1960         else
1961                 ret = map_bits;
1962  out:
1963         return ret;
1964 }
1965
1966 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1967 {
1968         unsigned int i;
1969
1970         if (!attrs)
1971                 return;
1972
1973         for (i = 0; i < attrs->n_assignments; i++)
1974                 kfree(attrs->assignment_str[i]);
1975
1976         for (i = 0; i < attrs->n_actions; i++)
1977                 kfree(attrs->action_str[i]);
1978
1979         kfree(attrs->name);
1980         kfree(attrs->sort_key_str);
1981         kfree(attrs->keys_str);
1982         kfree(attrs->vals_str);
1983         kfree(attrs->clock);
1984         kfree(attrs);
1985 }
1986
1987 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1988 {
1989         int ret = -EINVAL;
1990
1991         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1992                 return ret;
1993
1994         if ((str_has_prefix(str, "onmatch(")) ||
1995             (str_has_prefix(str, "onmax(")) ||
1996             (str_has_prefix(str, "onchange("))) {
1997                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1998                 if (!attrs->action_str[attrs->n_actions]) {
1999                         ret = -ENOMEM;
2000                         return ret;
2001                 }
2002                 attrs->n_actions++;
2003                 ret = 0;
2004         }
2005
2006         return ret;
2007 }
2008
2009 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
2010 {
2011         int ret = 0;
2012
2013         if ((str_has_prefix(str, "key=")) ||
2014             (str_has_prefix(str, "keys="))) {
2015                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2016                 if (!attrs->keys_str) {
2017                         ret = -ENOMEM;
2018                         goto out;
2019                 }
2020         } else if ((str_has_prefix(str, "val=")) ||
2021                    (str_has_prefix(str, "vals=")) ||
2022                    (str_has_prefix(str, "values="))) {
2023                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2024                 if (!attrs->vals_str) {
2025                         ret = -ENOMEM;
2026                         goto out;
2027                 }
2028         } else if (str_has_prefix(str, "sort=")) {
2029                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2030                 if (!attrs->sort_key_str) {
2031                         ret = -ENOMEM;
2032                         goto out;
2033                 }
2034         } else if (str_has_prefix(str, "name=")) {
2035                 attrs->name = kstrdup(str, GFP_KERNEL);
2036                 if (!attrs->name) {
2037                         ret = -ENOMEM;
2038                         goto out;
2039                 }
2040         } else if (str_has_prefix(str, "clock=")) {
2041                 strsep(&str, "=");
2042                 if (!str) {
2043                         ret = -EINVAL;
2044                         goto out;
2045                 }
2046
2047                 str = strstrip(str);
2048                 attrs->clock = kstrdup(str, GFP_KERNEL);
2049                 if (!attrs->clock) {
2050                         ret = -ENOMEM;
2051                         goto out;
2052                 }
2053         } else if (str_has_prefix(str, "size=")) {
2054                 int map_bits = parse_map_size(str);
2055
2056                 if (map_bits < 0) {
2057                         ret = map_bits;
2058                         goto out;
2059                 }
2060                 attrs->map_bits = map_bits;
2061         } else {
2062                 char *assignment;
2063
2064                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2065                         hist_err("Too many variables defined: ", str);
2066                         ret = -EINVAL;
2067                         goto out;
2068                 }
2069
2070                 assignment = kstrdup(str, GFP_KERNEL);
2071                 if (!assignment) {
2072                         ret = -ENOMEM;
2073                         goto out;
2074                 }
2075
2076                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2077         }
2078  out:
2079         return ret;
2080 }
2081
2082 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
2083 {
2084         struct hist_trigger_attrs *attrs;
2085         int ret = 0;
2086
2087         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2088         if (!attrs)
2089                 return ERR_PTR(-ENOMEM);
2090
2091         while (trigger_str) {
2092                 char *str = strsep(&trigger_str, ":");
2093
2094                 if (strchr(str, '=')) {
2095                         ret = parse_assignment(str, attrs);
2096                         if (ret)
2097                                 goto free;
2098                 } else if (strcmp(str, "pause") == 0)
2099                         attrs->pause = true;
2100                 else if ((strcmp(str, "cont") == 0) ||
2101                          (strcmp(str, "continue") == 0))
2102                         attrs->cont = true;
2103                 else if (strcmp(str, "clear") == 0)
2104                         attrs->clear = true;
2105                 else {
2106                         ret = parse_action(str, attrs);
2107                         if (ret)
2108                                 goto free;
2109                 }
2110         }
2111
2112         if (!attrs->keys_str) {
2113                 ret = -EINVAL;
2114                 goto free;
2115         }
2116
2117         if (!attrs->clock) {
2118                 attrs->clock = kstrdup("global", GFP_KERNEL);
2119                 if (!attrs->clock) {
2120                         ret = -ENOMEM;
2121                         goto free;
2122                 }
2123         }
2124
2125         return attrs;
2126  free:
2127         destroy_hist_trigger_attrs(attrs);
2128
2129         return ERR_PTR(ret);
2130 }
2131
2132 static inline void save_comm(char *comm, struct task_struct *task)
2133 {
2134         if (!task->pid) {
2135                 strcpy(comm, "<idle>");
2136                 return;
2137         }
2138
2139         if (WARN_ON_ONCE(task->pid < 0)) {
2140                 strcpy(comm, "<XXX>");
2141                 return;
2142         }
2143
2144         strncpy(comm, task->comm, TASK_COMM_LEN);
2145 }
2146
2147 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2148 {
2149         unsigned int i;
2150
2151         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2152                 kfree(elt_data->field_var_str[i]);
2153
2154         kfree(elt_data->comm);
2155         kfree(elt_data);
2156 }
2157
2158 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2159 {
2160         struct hist_elt_data *elt_data = elt->private_data;
2161
2162         hist_elt_data_free(elt_data);
2163 }
2164
2165 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2166 {
2167         struct hist_trigger_data *hist_data = elt->map->private_data;
2168         unsigned int size = TASK_COMM_LEN;
2169         struct hist_elt_data *elt_data;
2170         struct hist_field *key_field;
2171         unsigned int i, n_str;
2172
2173         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2174         if (!elt_data)
2175                 return -ENOMEM;
2176
2177         for_each_hist_key_field(i, hist_data) {
2178                 key_field = hist_data->fields[i];
2179
2180                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2181                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2182                         if (!elt_data->comm) {
2183                                 kfree(elt_data);
2184                                 return -ENOMEM;
2185                         }
2186                         break;
2187                 }
2188         }
2189
2190         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2191
2192         size = STR_VAR_LEN_MAX;
2193
2194         for (i = 0; i < n_str; i++) {
2195                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2196                 if (!elt_data->field_var_str[i]) {
2197                         hist_elt_data_free(elt_data);
2198                         return -ENOMEM;
2199                 }
2200         }
2201
2202         elt->private_data = elt_data;
2203
2204         return 0;
2205 }
2206
2207 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2208 {
2209         struct hist_elt_data *elt_data = elt->private_data;
2210
2211         if (elt_data->comm)
2212                 save_comm(elt_data->comm, current);
2213 }
2214
2215 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2216         .elt_alloc      = hist_trigger_elt_data_alloc,
2217         .elt_free       = hist_trigger_elt_data_free,
2218         .elt_init       = hist_trigger_elt_data_init,
2219 };
2220
2221 static const char *get_hist_field_flags(struct hist_field *hist_field)
2222 {
2223         const char *flags_str = NULL;
2224
2225         if (hist_field->flags & HIST_FIELD_FL_HEX)
2226                 flags_str = "hex";
2227         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2228                 flags_str = "sym";
2229         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2230                 flags_str = "sym-offset";
2231         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2232                 flags_str = "execname";
2233         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2234                 flags_str = "syscall";
2235         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2236                 flags_str = "log2";
2237         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2238                 flags_str = "usecs";
2239
2240         return flags_str;
2241 }
2242
2243 static void expr_field_str(struct hist_field *field, char *expr)
2244 {
2245         if (field->flags & HIST_FIELD_FL_VAR_REF)
2246                 strcat(expr, "$");
2247
2248         strcat(expr, hist_field_name(field, 0));
2249
2250         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2251                 const char *flags_str = get_hist_field_flags(field);
2252
2253                 if (flags_str) {
2254                         strcat(expr, ".");
2255                         strcat(expr, flags_str);
2256                 }
2257         }
2258 }
2259
2260 static char *expr_str(struct hist_field *field, unsigned int level)
2261 {
2262         char *expr;
2263
2264         if (level > 1)
2265                 return NULL;
2266
2267         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2268         if (!expr)
2269                 return NULL;
2270
2271         if (!field->operands[0]) {
2272                 expr_field_str(field, expr);
2273                 return expr;
2274         }
2275
2276         if (field->operator == FIELD_OP_UNARY_MINUS) {
2277                 char *subexpr;
2278
2279                 strcat(expr, "-(");
2280                 subexpr = expr_str(field->operands[0], ++level);
2281                 if (!subexpr) {
2282                         kfree(expr);
2283                         return NULL;
2284                 }
2285                 strcat(expr, subexpr);
2286                 strcat(expr, ")");
2287
2288                 kfree(subexpr);
2289
2290                 return expr;
2291         }
2292
2293         expr_field_str(field->operands[0], expr);
2294
2295         switch (field->operator) {
2296         case FIELD_OP_MINUS:
2297                 strcat(expr, "-");
2298                 break;
2299         case FIELD_OP_PLUS:
2300                 strcat(expr, "+");
2301                 break;
2302         default:
2303                 kfree(expr);
2304                 return NULL;
2305         }
2306
2307         expr_field_str(field->operands[1], expr);
2308
2309         return expr;
2310 }
2311
2312 static int contains_operator(char *str)
2313 {
2314         enum field_op_id field_op = FIELD_OP_NONE;
2315         char *op;
2316
2317         op = strpbrk(str, "+-");
2318         if (!op)
2319                 return FIELD_OP_NONE;
2320
2321         switch (*op) {
2322         case '-':
2323                 if (*str == '-')
2324                         field_op = FIELD_OP_UNARY_MINUS;
2325                 else
2326                         field_op = FIELD_OP_MINUS;
2327                 break;
2328         case '+':
2329                 field_op = FIELD_OP_PLUS;
2330                 break;
2331         default:
2332                 break;
2333         }
2334
2335         return field_op;
2336 }
2337
2338 static void __destroy_hist_field(struct hist_field *hist_field)
2339 {
2340         kfree(hist_field->var.name);
2341         kfree(hist_field->name);
2342         kfree(hist_field->type);
2343
2344         kfree(hist_field);
2345 }
2346
2347 static void destroy_hist_field(struct hist_field *hist_field,
2348                                unsigned int level)
2349 {
2350         unsigned int i;
2351
2352         if (level > 3)
2353                 return;
2354
2355         if (!hist_field)
2356                 return;
2357
2358         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2359                 return; /* var refs will be destroyed separately */
2360
2361         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2362                 destroy_hist_field(hist_field->operands[i], level + 1);
2363
2364         __destroy_hist_field(hist_field);
2365 }
2366
2367 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2368                                             struct ftrace_event_field *field,
2369                                             unsigned long flags,
2370                                             char *var_name)
2371 {
2372         struct hist_field *hist_field;
2373
2374         if (field && is_function_field(field))
2375                 return NULL;
2376
2377         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2378         if (!hist_field)
2379                 return NULL;
2380
2381         hist_field->hist_data = hist_data;
2382
2383         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2384                 goto out; /* caller will populate */
2385
2386         if (flags & HIST_FIELD_FL_VAR_REF) {
2387                 hist_field->fn = hist_field_var_ref;
2388                 goto out;
2389         }
2390
2391         if (flags & HIST_FIELD_FL_HITCOUNT) {
2392                 hist_field->fn = hist_field_counter;
2393                 hist_field->size = sizeof(u64);
2394                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2395                 if (!hist_field->type)
2396                         goto free;
2397                 goto out;
2398         }
2399
2400         if (flags & HIST_FIELD_FL_STACKTRACE) {
2401                 hist_field->fn = hist_field_none;
2402                 goto out;
2403         }
2404
2405         if (flags & HIST_FIELD_FL_LOG2) {
2406                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2407                 hist_field->fn = hist_field_log2;
2408                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2409                 hist_field->size = hist_field->operands[0]->size;
2410                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2411                 if (!hist_field->type)
2412                         goto free;
2413                 goto out;
2414         }
2415
2416         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2417                 hist_field->fn = hist_field_timestamp;
2418                 hist_field->size = sizeof(u64);
2419                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2420                 if (!hist_field->type)
2421                         goto free;
2422                 goto out;
2423         }
2424
2425         if (flags & HIST_FIELD_FL_CPU) {
2426                 hist_field->fn = hist_field_cpu;
2427                 hist_field->size = sizeof(int);
2428                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2429                 if (!hist_field->type)
2430                         goto free;
2431                 goto out;
2432         }
2433
2434         if (WARN_ON_ONCE(!field))
2435                 goto out;
2436
2437         if (is_string_field(field)) {
2438                 flags |= HIST_FIELD_FL_STRING;
2439
2440                 hist_field->size = MAX_FILTER_STR_VAL;
2441                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2442                 if (!hist_field->type)
2443                         goto free;
2444
2445                 if (field->filter_type == FILTER_STATIC_STRING)
2446                         hist_field->fn = hist_field_string;
2447                 else if (field->filter_type == FILTER_DYN_STRING)
2448                         hist_field->fn = hist_field_dynstring;
2449                 else
2450                         hist_field->fn = hist_field_pstring;
2451         } else {
2452                 hist_field->size = field->size;
2453                 hist_field->is_signed = field->is_signed;
2454                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2455                 if (!hist_field->type)
2456                         goto free;
2457
2458                 hist_field->fn = select_value_fn(field->size,
2459                                                  field->is_signed);
2460                 if (!hist_field->fn) {
2461                         destroy_hist_field(hist_field, 0);
2462                         return NULL;
2463                 }
2464         }
2465  out:
2466         hist_field->field = field;
2467         hist_field->flags = flags;
2468
2469         if (var_name) {
2470                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2471                 if (!hist_field->var.name)
2472                         goto free;
2473         }
2474
2475         return hist_field;
2476  free:
2477         destroy_hist_field(hist_field, 0);
2478         return NULL;
2479 }
2480
2481 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2482 {
2483         unsigned int i;
2484
2485         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2486                 if (hist_data->fields[i]) {
2487                         destroy_hist_field(hist_data->fields[i], 0);
2488                         hist_data->fields[i] = NULL;
2489                 }
2490         }
2491
2492         for (i = 0; i < hist_data->n_var_refs; i++) {
2493                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2494                 __destroy_hist_field(hist_data->var_refs[i]);
2495                 hist_data->var_refs[i] = NULL;
2496         }
2497 }
2498
2499 static int init_var_ref(struct hist_field *ref_field,
2500                         struct hist_field *var_field,
2501                         char *system, char *event_name)
2502 {
2503         int err = 0;
2504
2505         ref_field->var.idx = var_field->var.idx;
2506         ref_field->var.hist_data = var_field->hist_data;
2507         ref_field->size = var_field->size;
2508         ref_field->is_signed = var_field->is_signed;
2509         ref_field->flags |= var_field->flags &
2510                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2511
2512         if (system) {
2513                 ref_field->system = kstrdup(system, GFP_KERNEL);
2514                 if (!ref_field->system)
2515                         return -ENOMEM;
2516         }
2517
2518         if (event_name) {
2519                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2520                 if (!ref_field->event_name) {
2521                         err = -ENOMEM;
2522                         goto free;
2523                 }
2524         }
2525
2526         if (var_field->var.name) {
2527                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2528                 if (!ref_field->name) {
2529                         err = -ENOMEM;
2530                         goto free;
2531                 }
2532         } else if (var_field->name) {
2533                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2534                 if (!ref_field->name) {
2535                         err = -ENOMEM;
2536                         goto free;
2537                 }
2538         }
2539
2540         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2541         if (!ref_field->type) {
2542                 err = -ENOMEM;
2543                 goto free;
2544         }
2545  out:
2546         return err;
2547  free:
2548         kfree(ref_field->system);
2549         kfree(ref_field->event_name);
2550         kfree(ref_field->name);
2551
2552         goto out;
2553 }
2554
2555 /**
2556  * create_var_ref - Create a variable reference and attach it to trigger
2557  * @hist_data: The trigger that will be referencing the variable
2558  * @var_field: The VAR field to create a reference to
2559  * @system: The optional system string
2560  * @event_name: The optional event_name string
2561  *
2562  * Given a variable hist_field, create a VAR_REF hist_field that
2563  * represents a reference to it.
2564  *
2565  * This function also adds the reference to the trigger that
2566  * now references the variable.
2567  *
2568  * Return: The VAR_REF field if successful, NULL if not
2569  */
2570 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2571                                          struct hist_field *var_field,
2572                                          char *system, char *event_name)
2573 {
2574         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2575         struct hist_field *ref_field;
2576
2577         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2578         if (ref_field) {
2579                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2580                         destroy_hist_field(ref_field, 0);
2581                         return NULL;
2582                 }
2583
2584                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2585                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2586         }
2587
2588         return ref_field;
2589 }
2590
2591 static bool is_var_ref(char *var_name)
2592 {
2593         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2594                 return false;
2595
2596         return true;
2597 }
2598
2599 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2600                                  char *var_name)
2601 {
2602         char *name, *field;
2603         unsigned int i;
2604
2605         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2606                 name = hist_data->attrs->var_defs.name[i];
2607
2608                 if (strcmp(var_name, name) == 0) {
2609                         field = hist_data->attrs->var_defs.expr[i];
2610                         if (contains_operator(field) || is_var_ref(field))
2611                                 continue;
2612                         return field;
2613                 }
2614         }
2615
2616         return NULL;
2617 }
2618
2619 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2620                                  char *system, char *event_name,
2621                                  char *var_name)
2622 {
2623         struct trace_event_call *call;
2624
2625         if (system && event_name) {
2626                 call = hist_data->event_file->event_call;
2627
2628                 if (strcmp(system, call->class->system) != 0)
2629                         return NULL;
2630
2631                 if (strcmp(event_name, trace_event_name(call)) != 0)
2632                         return NULL;
2633         }
2634
2635         if (!!system != !!event_name)
2636                 return NULL;
2637
2638         if (!is_var_ref(var_name))
2639                 return NULL;
2640
2641         var_name++;
2642
2643         return field_name_from_var(hist_data, var_name);
2644 }
2645
2646 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2647                                         char *system, char *event_name,
2648                                         char *var_name)
2649 {
2650         struct hist_field *var_field = NULL, *ref_field = NULL;
2651
2652         if (!is_var_ref(var_name))
2653                 return NULL;
2654
2655         var_name++;
2656
2657         var_field = find_event_var(hist_data, system, event_name, var_name);
2658         if (var_field)
2659                 ref_field = create_var_ref(hist_data, var_field,
2660                                            system, event_name);
2661
2662         if (!ref_field)
2663                 hist_err_event("Couldn't find variable: $",
2664                                system, event_name, var_name);
2665
2666         return ref_field;
2667 }
2668
2669 static struct ftrace_event_field *
2670 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2671             char *field_str, unsigned long *flags)
2672 {
2673         struct ftrace_event_field *field = NULL;
2674         char *field_name, *modifier, *str;
2675
2676         modifier = str = kstrdup(field_str, GFP_KERNEL);
2677         if (!modifier)
2678                 return ERR_PTR(-ENOMEM);
2679
2680         field_name = strsep(&modifier, ".");
2681         if (modifier) {
2682                 if (strcmp(modifier, "hex") == 0)
2683                         *flags |= HIST_FIELD_FL_HEX;
2684                 else if (strcmp(modifier, "sym") == 0)
2685                         *flags |= HIST_FIELD_FL_SYM;
2686                 else if (strcmp(modifier, "sym-offset") == 0)
2687                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2688                 else if ((strcmp(modifier, "execname") == 0) &&
2689                          (strcmp(field_name, "common_pid") == 0))
2690                         *flags |= HIST_FIELD_FL_EXECNAME;
2691                 else if (strcmp(modifier, "syscall") == 0)
2692                         *flags |= HIST_FIELD_FL_SYSCALL;
2693                 else if (strcmp(modifier, "log2") == 0)
2694                         *flags |= HIST_FIELD_FL_LOG2;
2695                 else if (strcmp(modifier, "usecs") == 0)
2696                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2697                 else {
2698                         hist_err("Invalid field modifier: ", modifier);
2699                         field = ERR_PTR(-EINVAL);
2700                         goto out;
2701                 }
2702         }
2703
2704         if (strcmp(field_name, "common_timestamp") == 0) {
2705                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2706                 hist_data->enable_timestamps = true;
2707                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2708                         hist_data->attrs->ts_in_usecs = true;
2709         } else if (strcmp(field_name, "cpu") == 0)
2710                 *flags |= HIST_FIELD_FL_CPU;
2711         else {
2712                 field = trace_find_event_field(file->event_call, field_name);
2713                 if (!field || !field->size) {
2714                         hist_err("Couldn't find field: ", field_name);
2715                         field = ERR_PTR(-EINVAL);
2716                         goto out;
2717                 }
2718         }
2719  out:
2720         kfree(str);
2721
2722         return field;
2723 }
2724
2725 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2726                                        struct hist_field *var_ref,
2727                                        char *var_name)
2728 {
2729         struct hist_field *alias = NULL;
2730         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2731
2732         alias = create_hist_field(hist_data, NULL, flags, var_name);
2733         if (!alias)
2734                 return NULL;
2735
2736         alias->fn = var_ref->fn;
2737         alias->operands[0] = var_ref;
2738
2739         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2740                 destroy_hist_field(alias, 0);
2741                 return NULL;
2742         }
2743
2744         return alias;
2745 }
2746
2747 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2748                                      struct trace_event_file *file, char *str,
2749                                      unsigned long *flags, char *var_name)
2750 {
2751         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2752         struct ftrace_event_field *field = NULL;
2753         struct hist_field *hist_field = NULL;
2754         int ret = 0;
2755
2756         s = strchr(str, '.');
2757         if (s) {
2758                 s = strchr(++s, '.');
2759                 if (s) {
2760                         ref_system = strsep(&str, ".");
2761                         if (!str) {
2762                                 ret = -EINVAL;
2763                                 goto out;
2764                         }
2765                         ref_event = strsep(&str, ".");
2766                         if (!str) {
2767                                 ret = -EINVAL;
2768                                 goto out;
2769                         }
2770                         ref_var = str;
2771                 }
2772         }
2773
2774         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2775         if (!s) {
2776                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2777                 if (hist_field) {
2778                         if (var_name) {
2779                                 hist_field = create_alias(hist_data, hist_field, var_name);
2780                                 if (!hist_field) {
2781                                         ret = -ENOMEM;
2782                                         goto out;
2783                                 }
2784                         }
2785                         return hist_field;
2786                 }
2787         } else
2788                 str = s;
2789
2790         field = parse_field(hist_data, file, str, flags);
2791         if (IS_ERR(field)) {
2792                 ret = PTR_ERR(field);
2793                 goto out;
2794         }
2795
2796         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2797         if (!hist_field) {
2798                 ret = -ENOMEM;
2799                 goto out;
2800         }
2801
2802         return hist_field;
2803  out:
2804         return ERR_PTR(ret);
2805 }
2806
2807 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2808                                      struct trace_event_file *file,
2809                                      char *str, unsigned long flags,
2810                                      char *var_name, unsigned int level);
2811
2812 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2813                                       struct trace_event_file *file,
2814                                       char *str, unsigned long flags,
2815                                       char *var_name, unsigned int level)
2816 {
2817         struct hist_field *operand1, *expr = NULL;
2818         unsigned long operand_flags;
2819         int ret = 0;
2820         char *s;
2821
2822         /* we support only -(xxx) i.e. explicit parens required */
2823
2824         if (level > 3) {
2825                 hist_err("Too many subexpressions (3 max): ", str);
2826                 ret = -EINVAL;
2827                 goto free;
2828         }
2829
2830         str++; /* skip leading '-' */
2831
2832         s = strchr(str, '(');
2833         if (s)
2834                 str++;
2835         else {
2836                 ret = -EINVAL;
2837                 goto free;
2838         }
2839
2840         s = strrchr(str, ')');
2841         if (s)
2842                 *s = '\0';
2843         else {
2844                 ret = -EINVAL; /* no closing ')' */
2845                 goto free;
2846         }
2847
2848         flags |= HIST_FIELD_FL_EXPR;
2849         expr = create_hist_field(hist_data, NULL, flags, var_name);
2850         if (!expr) {
2851                 ret = -ENOMEM;
2852                 goto free;
2853         }
2854
2855         operand_flags = 0;
2856         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2857         if (IS_ERR(operand1)) {
2858                 ret = PTR_ERR(operand1);
2859                 goto free;
2860         }
2861
2862         expr->flags |= operand1->flags &
2863                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2864         expr->fn = hist_field_unary_minus;
2865         expr->operands[0] = operand1;
2866         expr->operator = FIELD_OP_UNARY_MINUS;
2867         expr->name = expr_str(expr, 0);
2868         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2869         if (!expr->type) {
2870                 ret = -ENOMEM;
2871                 goto free;
2872         }
2873
2874         return expr;
2875  free:
2876         destroy_hist_field(expr, 0);
2877         return ERR_PTR(ret);
2878 }
2879
2880 static int check_expr_operands(struct hist_field *operand1,
2881                                struct hist_field *operand2)
2882 {
2883         unsigned long operand1_flags = operand1->flags;
2884         unsigned long operand2_flags = operand2->flags;
2885
2886         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2887             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2888                 struct hist_field *var;
2889
2890                 var = find_var_field(operand1->var.hist_data, operand1->name);
2891                 if (!var)
2892                         return -EINVAL;
2893                 operand1_flags = var->flags;
2894         }
2895
2896         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2897             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2898                 struct hist_field *var;
2899
2900                 var = find_var_field(operand2->var.hist_data, operand2->name);
2901                 if (!var)
2902                         return -EINVAL;
2903                 operand2_flags = var->flags;
2904         }
2905
2906         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2907             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2908                 hist_err("Timestamp units in expression don't match", NULL);
2909                 return -EINVAL;
2910         }
2911
2912         return 0;
2913 }
2914
2915 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2916                                      struct trace_event_file *file,
2917                                      char *str, unsigned long flags,
2918                                      char *var_name, unsigned int level)
2919 {
2920         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2921         unsigned long operand_flags;
2922         int field_op, ret = -EINVAL;
2923         char *sep, *operand1_str;
2924
2925         if (level > 3) {
2926                 hist_err("Too many subexpressions (3 max): ", str);
2927                 return ERR_PTR(-EINVAL);
2928         }
2929
2930         field_op = contains_operator(str);
2931
2932         if (field_op == FIELD_OP_NONE)
2933                 return parse_atom(hist_data, file, str, &flags, var_name);
2934
2935         if (field_op == FIELD_OP_UNARY_MINUS)
2936                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2937
2938         switch (field_op) {
2939         case FIELD_OP_MINUS:
2940                 sep = "-";
2941                 break;
2942         case FIELD_OP_PLUS:
2943                 sep = "+";
2944                 break;
2945         default:
2946                 goto free;
2947         }
2948
2949         operand1_str = strsep(&str, sep);
2950         if (!operand1_str || !str)
2951                 goto free;
2952
2953         operand_flags = 0;
2954         operand1 = parse_atom(hist_data, file, operand1_str,
2955                               &operand_flags, NULL);
2956         if (IS_ERR(operand1)) {
2957                 ret = PTR_ERR(operand1);
2958                 operand1 = NULL;
2959                 goto free;
2960         }
2961
2962         /* rest of string could be another expression e.g. b+c in a+b+c */
2963         operand_flags = 0;
2964         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2965         if (IS_ERR(operand2)) {
2966                 ret = PTR_ERR(operand2);
2967                 operand2 = NULL;
2968                 goto free;
2969         }
2970
2971         ret = check_expr_operands(operand1, operand2);
2972         if (ret)
2973                 goto free;
2974
2975         flags |= HIST_FIELD_FL_EXPR;
2976
2977         flags |= operand1->flags &
2978                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2979
2980         expr = create_hist_field(hist_data, NULL, flags, var_name);
2981         if (!expr) {
2982                 ret = -ENOMEM;
2983                 goto free;
2984         }
2985
2986         operand1->read_once = true;
2987         operand2->read_once = true;
2988
2989         expr->operands[0] = operand1;
2990         expr->operands[1] = operand2;
2991         expr->operator = field_op;
2992         expr->name = expr_str(expr, 0);
2993         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2994         if (!expr->type) {
2995                 ret = -ENOMEM;
2996                 goto free;
2997         }
2998
2999         switch (field_op) {
3000         case FIELD_OP_MINUS:
3001                 expr->fn = hist_field_minus;
3002                 break;
3003         case FIELD_OP_PLUS:
3004                 expr->fn = hist_field_plus;
3005                 break;
3006         default:
3007                 ret = -EINVAL;
3008                 goto free;
3009         }
3010
3011         return expr;
3012  free:
3013         destroy_hist_field(operand1, 0);
3014         destroy_hist_field(operand2, 0);
3015         destroy_hist_field(expr, 0);
3016
3017         return ERR_PTR(ret);
3018 }
3019
3020 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3021                                  struct trace_event_file *file)
3022 {
3023         struct event_trigger_data *test;
3024
3025         list_for_each_entry_rcu(test, &file->triggers, list) {
3026                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3027                         if (test->private_data == hist_data)
3028                                 return test->filter_str;
3029                 }
3030         }
3031
3032         return NULL;
3033 }
3034
3035 static struct event_command trigger_hist_cmd;
3036 static int event_hist_trigger_func(struct event_command *cmd_ops,
3037                                    struct trace_event_file *file,
3038                                    char *glob, char *cmd, char *param);
3039
3040 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3041                             struct hist_trigger_data *hist_data,
3042                             unsigned int n_keys)
3043 {
3044         struct hist_field *target_hist_field, *hist_field;
3045         unsigned int n, i, j;
3046
3047         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3048                 return false;
3049
3050         i = hist_data->n_vals;
3051         j = target_hist_data->n_vals;
3052
3053         for (n = 0; n < n_keys; n++) {
3054                 hist_field = hist_data->fields[i + n];
3055                 target_hist_field = target_hist_data->fields[j + n];
3056
3057                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3058                         return false;
3059                 if (hist_field->size != target_hist_field->size)
3060                         return false;
3061                 if (hist_field->is_signed != target_hist_field->is_signed)
3062                         return false;
3063         }
3064
3065         return true;
3066 }
3067
3068 static struct hist_trigger_data *
3069 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3070                      struct trace_event_file *file)
3071 {
3072         struct hist_trigger_data *hist_data;
3073         struct event_trigger_data *test;
3074         unsigned int n_keys;
3075
3076         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3077
3078         list_for_each_entry_rcu(test, &file->triggers, list) {
3079                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3080                         hist_data = test->private_data;
3081
3082                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3083                                 return hist_data;
3084                 }
3085         }
3086
3087         return NULL;
3088 }
3089
3090 static struct trace_event_file *event_file(struct trace_array *tr,
3091                                            char *system, char *event_name)
3092 {
3093         struct trace_event_file *file;
3094
3095         file = __find_event_file(tr, system, event_name);
3096         if (!file)
3097                 return ERR_PTR(-EINVAL);
3098
3099         return file;
3100 }
3101
3102 static struct hist_field *
3103 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3104                          char *system, char *event_name, char *field_name)
3105 {
3106         struct hist_field *event_var;
3107         char *synthetic_name;
3108
3109         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3110         if (!synthetic_name)
3111                 return ERR_PTR(-ENOMEM);
3112
3113         strcpy(synthetic_name, "synthetic_");
3114         strcat(synthetic_name, field_name);
3115
3116         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3117
3118         kfree(synthetic_name);
3119
3120         return event_var;
3121 }
3122
3123 /**
3124  * create_field_var_hist - Automatically create a histogram and var for a field
3125  * @target_hist_data: The target hist trigger
3126  * @subsys_name: Optional subsystem name
3127  * @event_name: Optional event name
3128  * @field_name: The name of the field (and the resulting variable)
3129  *
3130  * Hist trigger actions fetch data from variables, not directly from
3131  * events.  However, for convenience, users are allowed to directly
3132  * specify an event field in an action, which will be automatically
3133  * converted into a variable on their behalf.
3134
3135  * If a user specifies a field on an event that isn't the event the
3136  * histogram currently being defined (the target event histogram), the
3137  * only way that can be accomplished is if a new hist trigger is
3138  * created and the field variable defined on that.
3139  *
3140  * This function creates a new histogram compatible with the target
3141  * event (meaning a histogram with the same key as the target
3142  * histogram), and creates a variable for the specified field, but
3143  * with 'synthetic_' prepended to the variable name in order to avoid
3144  * collision with normal field variables.
3145  *
3146  * Return: The variable created for the field.
3147  */
3148 static struct hist_field *
3149 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3150                       char *subsys_name, char *event_name, char *field_name)
3151 {
3152         struct trace_array *tr = target_hist_data->event_file->tr;
3153         struct hist_field *event_var = ERR_PTR(-EINVAL);
3154         struct hist_trigger_data *hist_data;
3155         unsigned int i, n, first = true;
3156         struct field_var_hist *var_hist;
3157         struct trace_event_file *file;
3158         struct hist_field *key_field;
3159         char *saved_filter;
3160         char *cmd;
3161         int ret;
3162
3163         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3164                 hist_err_event("trace action: Too many field variables defined: ",
3165                                subsys_name, event_name, field_name);
3166                 return ERR_PTR(-EINVAL);
3167         }
3168
3169         file = event_file(tr, subsys_name, event_name);
3170
3171         if (IS_ERR(file)) {
3172                 hist_err_event("trace action: Event file not found: ",
3173                                subsys_name, event_name, field_name);
3174                 ret = PTR_ERR(file);
3175                 return ERR_PTR(ret);
3176         }
3177
3178         /*
3179          * Look for a histogram compatible with target.  We'll use the
3180          * found histogram specification to create a new matching
3181          * histogram with our variable on it.  target_hist_data is not
3182          * yet a registered histogram so we can't use that.
3183          */
3184         hist_data = find_compatible_hist(target_hist_data, file);
3185         if (!hist_data) {
3186                 hist_err_event("trace action: Matching event histogram not found: ",
3187                                subsys_name, event_name, field_name);
3188                 return ERR_PTR(-EINVAL);
3189         }
3190
3191         /* See if a synthetic field variable has already been created */
3192         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3193                                              event_name, field_name);
3194         if (!IS_ERR_OR_NULL(event_var))
3195                 return event_var;
3196
3197         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3198         if (!var_hist)
3199                 return ERR_PTR(-ENOMEM);
3200
3201         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3202         if (!cmd) {
3203                 kfree(var_hist);
3204                 return ERR_PTR(-ENOMEM);
3205         }
3206
3207         /* Use the same keys as the compatible histogram */
3208         strcat(cmd, "keys=");
3209
3210         for_each_hist_key_field(i, hist_data) {
3211                 key_field = hist_data->fields[i];
3212                 if (!first)
3213                         strcat(cmd, ",");
3214                 strcat(cmd, key_field->field->name);
3215                 first = false;
3216         }
3217
3218         /* Create the synthetic field variable specification */
3219         strcat(cmd, ":synthetic_");
3220         strcat(cmd, field_name);
3221         strcat(cmd, "=");
3222         strcat(cmd, field_name);
3223
3224         /* Use the same filter as the compatible histogram */
3225         saved_filter = find_trigger_filter(hist_data, file);
3226         if (saved_filter) {
3227                 strcat(cmd, " if ");
3228                 strcat(cmd, saved_filter);
3229         }
3230
3231         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3232         if (!var_hist->cmd) {
3233                 kfree(cmd);
3234                 kfree(var_hist);
3235                 return ERR_PTR(-ENOMEM);
3236         }
3237
3238         /* Save the compatible histogram information */
3239         var_hist->hist_data = hist_data;
3240
3241         /* Create the new histogram with our variable */
3242         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3243                                       "", "hist", cmd);
3244         if (ret) {
3245                 kfree(cmd);
3246                 kfree(var_hist->cmd);
3247                 kfree(var_hist);
3248                 hist_err_event("trace action: Couldn't create histogram for field: ",
3249                                subsys_name, event_name, field_name);
3250                 return ERR_PTR(ret);
3251         }
3252
3253         kfree(cmd);
3254
3255         /* If we can't find the variable, something went wrong */
3256         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3257                                              event_name, field_name);
3258         if (IS_ERR_OR_NULL(event_var)) {
3259                 kfree(var_hist->cmd);
3260                 kfree(var_hist);
3261                 hist_err_event("trace action: Couldn't find synthetic variable: ",
3262                                subsys_name, event_name, field_name);
3263                 return ERR_PTR(-EINVAL);
3264         }
3265
3266         n = target_hist_data->n_field_var_hists;
3267         target_hist_data->field_var_hists[n] = var_hist;
3268         target_hist_data->n_field_var_hists++;
3269
3270         return event_var;
3271 }
3272
3273 static struct hist_field *
3274 find_target_event_var(struct hist_trigger_data *hist_data,
3275                       char *subsys_name, char *event_name, char *var_name)
3276 {
3277         struct trace_event_file *file = hist_data->event_file;
3278         struct hist_field *hist_field = NULL;
3279
3280         if (subsys_name) {
3281                 struct trace_event_call *call;
3282
3283                 if (!event_name)
3284                         return NULL;
3285
3286                 call = file->event_call;
3287
3288                 if (strcmp(subsys_name, call->class->system) != 0)
3289                         return NULL;
3290
3291                 if (strcmp(event_name, trace_event_name(call)) != 0)
3292                         return NULL;
3293         }
3294
3295         hist_field = find_var_field(hist_data, var_name);
3296
3297         return hist_field;
3298 }
3299
3300 static inline void __update_field_vars(struct tracing_map_elt *elt,
3301                                        struct ring_buffer_event *rbe,
3302                                        void *rec,
3303                                        struct field_var **field_vars,
3304                                        unsigned int n_field_vars,
3305                                        unsigned int field_var_str_start)
3306 {
3307         struct hist_elt_data *elt_data = elt->private_data;
3308         unsigned int i, j, var_idx;
3309         u64 var_val;
3310
3311         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3312                 struct field_var *field_var = field_vars[i];
3313                 struct hist_field *var = field_var->var;
3314                 struct hist_field *val = field_var->val;
3315
3316                 var_val = val->fn(val, elt, rbe, rec);
3317                 var_idx = var->var.idx;
3318
3319                 if (val->flags & HIST_FIELD_FL_STRING) {
3320                         char *str = elt_data->field_var_str[j++];
3321                         char *val_str = (char *)(uintptr_t)var_val;
3322
3323                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3324                         var_val = (u64)(uintptr_t)str;
3325                 }
3326                 tracing_map_set_var(elt, var_idx, var_val);
3327         }
3328 }
3329
3330 static void update_field_vars(struct hist_trigger_data *hist_data,
3331                               struct tracing_map_elt *elt,