301f224304698950544088c16518ea2e14ff41a6
[muen/linux.git] / net / tls / tls_main.c
1 /*
2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #include <linux/module.h>
35
36 #include <net/tcp.h>
37 #include <net/inet_common.h>
38 #include <linux/highmem.h>
39 #include <linux/netdevice.h>
40 #include <linux/sched/signal.h>
41 #include <linux/inetdevice.h>
42
43 #include <net/tls.h>
44
45 MODULE_AUTHOR("Mellanox Technologies");
46 MODULE_DESCRIPTION("Transport Layer Security Support");
47 MODULE_LICENSE("Dual BSD/GPL");
48
49 enum {
50         TLSV4,
51         TLSV6,
52         TLS_NUM_PROTS,
53 };
54 enum {
55         TLS_BASE,
56         TLS_SW,
57 #ifdef CONFIG_TLS_DEVICE
58         TLS_HW,
59 #endif
60         TLS_HW_RECORD,
61         TLS_NUM_CONFIG,
62 };
63
64 static struct proto *saved_tcpv6_prot;
65 static DEFINE_MUTEX(tcpv6_prot_mutex);
66 static LIST_HEAD(device_list);
67 static DEFINE_MUTEX(device_mutex);
68 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
69 static struct proto_ops tls_sw_proto_ops;
70
71 static void update_sk_prot(struct sock *sk, struct tls_context *ctx)
72 {
73         int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
74
75         sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf];
76 }
77
78 int wait_on_pending_writer(struct sock *sk, long *timeo)
79 {
80         int rc = 0;
81         DEFINE_WAIT_FUNC(wait, woken_wake_function);
82
83         add_wait_queue(sk_sleep(sk), &wait);
84         while (1) {
85                 if (!*timeo) {
86                         rc = -EAGAIN;
87                         break;
88                 }
89
90                 if (signal_pending(current)) {
91                         rc = sock_intr_errno(*timeo);
92                         break;
93                 }
94
95                 if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
96                         break;
97         }
98         remove_wait_queue(sk_sleep(sk), &wait);
99         return rc;
100 }
101
102 int tls_push_sg(struct sock *sk,
103                 struct tls_context *ctx,
104                 struct scatterlist *sg,
105                 u16 first_offset,
106                 int flags)
107 {
108         int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST;
109         int ret = 0;
110         struct page *p;
111         size_t size;
112         int offset = first_offset;
113
114         size = sg->length - offset;
115         offset += sg->offset;
116
117         ctx->in_tcp_sendpages = true;
118         while (1) {
119                 if (sg_is_last(sg))
120                         sendpage_flags = flags;
121
122                 /* is sending application-limited? */
123                 tcp_rate_check_app_limited(sk);
124                 p = sg_page(sg);
125 retry:
126                 ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags);
127
128                 if (ret != size) {
129                         if (ret > 0) {
130                                 offset += ret;
131                                 size -= ret;
132                                 goto retry;
133                         }
134
135                         offset -= sg->offset;
136                         ctx->partially_sent_offset = offset;
137                         ctx->partially_sent_record = (void *)sg;
138                         ctx->in_tcp_sendpages = false;
139                         return ret;
140                 }
141
142                 put_page(p);
143                 sk_mem_uncharge(sk, sg->length);
144                 sg = sg_next(sg);
145                 if (!sg)
146                         break;
147
148                 offset = sg->offset;
149                 size = sg->length;
150         }
151
152         clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
153         ctx->in_tcp_sendpages = false;
154         ctx->sk_write_space(sk);
155
156         return 0;
157 }
158
159 static int tls_handle_open_record(struct sock *sk, int flags)
160 {
161         struct tls_context *ctx = tls_get_ctx(sk);
162
163         if (tls_is_pending_open_record(ctx))
164                 return ctx->push_pending_record(sk, flags);
165
166         return 0;
167 }
168
169 int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
170                       unsigned char *record_type)
171 {
172         struct cmsghdr *cmsg;
173         int rc = -EINVAL;
174
175         for_each_cmsghdr(cmsg, msg) {
176                 if (!CMSG_OK(msg, cmsg))
177                         return -EINVAL;
178                 if (cmsg->cmsg_level != SOL_TLS)
179                         continue;
180
181                 switch (cmsg->cmsg_type) {
182                 case TLS_SET_RECORD_TYPE:
183                         if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
184                                 return -EINVAL;
185
186                         if (msg->msg_flags & MSG_MORE)
187                                 return -EINVAL;
188
189                         rc = tls_handle_open_record(sk, msg->msg_flags);
190                         if (rc)
191                                 return rc;
192
193                         *record_type = *(unsigned char *)CMSG_DATA(cmsg);
194                         rc = 0;
195                         break;
196                 default:
197                         return -EINVAL;
198                 }
199         }
200
201         return rc;
202 }
203
204 int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx,
205                                    int flags, long *timeo)
206 {
207         struct scatterlist *sg;
208         u16 offset;
209
210         if (!tls_is_partially_sent_record(ctx))
211                 return ctx->push_pending_record(sk, flags);
212
213         sg = ctx->partially_sent_record;
214         offset = ctx->partially_sent_offset;
215
216         ctx->partially_sent_record = NULL;
217         return tls_push_sg(sk, ctx, sg, offset, flags);
218 }
219
220 static void tls_write_space(struct sock *sk)
221 {
222         struct tls_context *ctx = tls_get_ctx(sk);
223
224         /* We are already sending pages, ignore notification */
225         if (ctx->in_tcp_sendpages)
226                 return;
227
228         if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
229                 gfp_t sk_allocation = sk->sk_allocation;
230                 int rc;
231                 long timeo = 0;
232
233                 sk->sk_allocation = GFP_ATOMIC;
234                 rc = tls_push_pending_closed_record(sk, ctx,
235                                                     MSG_DONTWAIT |
236                                                     MSG_NOSIGNAL,
237                                                     &timeo);
238                 sk->sk_allocation = sk_allocation;
239
240                 if (rc < 0)
241                         return;
242         }
243
244         ctx->sk_write_space(sk);
245 }
246
247 static void tls_sk_proto_close(struct sock *sk, long timeout)
248 {
249         struct tls_context *ctx = tls_get_ctx(sk);
250         long timeo = sock_sndtimeo(sk, 0);
251         void (*sk_proto_close)(struct sock *sk, long timeout);
252         bool free_ctx = false;
253
254         lock_sock(sk);
255         sk_proto_close = ctx->sk_proto_close;
256
257         if ((ctx->tx_conf == TLS_HW_RECORD && ctx->rx_conf == TLS_HW_RECORD) ||
258             (ctx->tx_conf == TLS_BASE && ctx->rx_conf == TLS_BASE)) {
259                 free_ctx = true;
260                 goto skip_tx_cleanup;
261         }
262
263         if (!tls_complete_pending_work(sk, ctx, 0, &timeo))
264                 tls_handle_open_record(sk, 0);
265
266         if (ctx->partially_sent_record) {
267                 struct scatterlist *sg = ctx->partially_sent_record;
268
269                 while (1) {
270                         put_page(sg_page(sg));
271                         sk_mem_uncharge(sk, sg->length);
272
273                         if (sg_is_last(sg))
274                                 break;
275                         sg++;
276                 }
277         }
278
279         /* We need these for tls_sw_fallback handling of other packets */
280         if (ctx->tx_conf == TLS_SW) {
281                 kfree(ctx->tx.rec_seq);
282                 kfree(ctx->tx.iv);
283                 tls_sw_free_resources_tx(sk);
284         }
285
286         if (ctx->rx_conf == TLS_SW) {
287                 kfree(ctx->rx.rec_seq);
288                 kfree(ctx->rx.iv);
289                 tls_sw_free_resources_rx(sk);
290         }
291
292 #ifdef CONFIG_TLS_DEVICE
293         if (ctx->tx_conf != TLS_HW) {
294 #else
295         {
296 #endif
297                 kfree(ctx);
298                 ctx = NULL;
299         }
300
301 skip_tx_cleanup:
302         release_sock(sk);
303         sk_proto_close(sk, timeout);
304         /* free ctx for TLS_HW_RECORD, used by tcp_set_state
305          * for sk->sk_prot->unhash [tls_hw_unhash]
306          */
307         if (free_ctx)
308                 kfree(ctx);
309 }
310
311 static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
312                                 int __user *optlen)
313 {
314         int rc = 0;
315         struct tls_context *ctx = tls_get_ctx(sk);
316         struct tls_crypto_info *crypto_info;
317         int len;
318
319         if (get_user(len, optlen))
320                 return -EFAULT;
321
322         if (!optval || (len < sizeof(*crypto_info))) {
323                 rc = -EINVAL;
324                 goto out;
325         }
326
327         if (!ctx) {
328                 rc = -EBUSY;
329                 goto out;
330         }
331
332         /* get user crypto info */
333         crypto_info = &ctx->crypto_send;
334
335         if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
336                 rc = -EBUSY;
337                 goto out;
338         }
339
340         if (len == sizeof(*crypto_info)) {
341                 if (copy_to_user(optval, crypto_info, sizeof(*crypto_info)))
342                         rc = -EFAULT;
343                 goto out;
344         }
345
346         switch (crypto_info->cipher_type) {
347         case TLS_CIPHER_AES_GCM_128: {
348                 struct tls12_crypto_info_aes_gcm_128 *
349                   crypto_info_aes_gcm_128 =
350                   container_of(crypto_info,
351                                struct tls12_crypto_info_aes_gcm_128,
352                                info);
353
354                 if (len != sizeof(*crypto_info_aes_gcm_128)) {
355                         rc = -EINVAL;
356                         goto out;
357                 }
358                 lock_sock(sk);
359                 memcpy(crypto_info_aes_gcm_128->iv,
360                        ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
361                        TLS_CIPHER_AES_GCM_128_IV_SIZE);
362                 memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->tx.rec_seq,
363                        TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
364                 release_sock(sk);
365                 if (copy_to_user(optval,
366                                  crypto_info_aes_gcm_128,
367                                  sizeof(*crypto_info_aes_gcm_128)))
368                         rc = -EFAULT;
369                 break;
370         }
371         default:
372                 rc = -EINVAL;
373         }
374
375 out:
376         return rc;
377 }
378
379 static int do_tls_getsockopt(struct sock *sk, int optname,
380                              char __user *optval, int __user *optlen)
381 {
382         int rc = 0;
383
384         switch (optname) {
385         case TLS_TX:
386                 rc = do_tls_getsockopt_tx(sk, optval, optlen);
387                 break;
388         default:
389                 rc = -ENOPROTOOPT;
390                 break;
391         }
392         return rc;
393 }
394
395 static int tls_getsockopt(struct sock *sk, int level, int optname,
396                           char __user *optval, int __user *optlen)
397 {
398         struct tls_context *ctx = tls_get_ctx(sk);
399
400         if (level != SOL_TLS)
401                 return ctx->getsockopt(sk, level, optname, optval, optlen);
402
403         return do_tls_getsockopt(sk, optname, optval, optlen);
404 }
405
406 static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval,
407                                   unsigned int optlen, int tx)
408 {
409         struct tls_crypto_info *crypto_info;
410         struct tls_context *ctx = tls_get_ctx(sk);
411         int rc = 0;
412         int conf;
413
414         if (!optval || (optlen < sizeof(*crypto_info))) {
415                 rc = -EINVAL;
416                 goto out;
417         }
418
419         if (tx)
420                 crypto_info = &ctx->crypto_send;
421         else
422                 crypto_info = &ctx->crypto_recv;
423
424         /* Currently we don't support set crypto info more than one time */
425         if (TLS_CRYPTO_INFO_READY(crypto_info)) {
426                 rc = -EBUSY;
427                 goto out;
428         }
429
430         rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info));
431         if (rc) {
432                 rc = -EFAULT;
433                 goto err_crypto_info;
434         }
435
436         /* check version */
437         if (crypto_info->version != TLS_1_2_VERSION) {
438                 rc = -ENOTSUPP;
439                 goto err_crypto_info;
440         }
441
442         switch (crypto_info->cipher_type) {
443         case TLS_CIPHER_AES_GCM_128: {
444                 if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) {
445                         rc = -EINVAL;
446                         goto err_crypto_info;
447                 }
448                 rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info),
449                                     optlen - sizeof(*crypto_info));
450                 if (rc) {
451                         rc = -EFAULT;
452                         goto err_crypto_info;
453                 }
454                 break;
455         }
456         default:
457                 rc = -EINVAL;
458                 goto err_crypto_info;
459         }
460
461         if (tx) {
462 #ifdef CONFIG_TLS_DEVICE
463                 rc = tls_set_device_offload(sk, ctx);
464                 conf = TLS_HW;
465                 if (rc) {
466 #else
467                 {
468 #endif
469                         rc = tls_set_sw_offload(sk, ctx, 1);
470                         conf = TLS_SW;
471                 }
472         } else {
473                 rc = tls_set_sw_offload(sk, ctx, 0);
474                 conf = TLS_SW;
475         }
476
477         if (rc)
478                 goto err_crypto_info;
479
480         if (tx)
481                 ctx->tx_conf = conf;
482         else
483                 ctx->rx_conf = conf;
484         update_sk_prot(sk, ctx);
485         if (tx) {
486                 ctx->sk_write_space = sk->sk_write_space;
487                 sk->sk_write_space = tls_write_space;
488         } else {
489                 sk->sk_socket->ops = &tls_sw_proto_ops;
490         }
491         goto out;
492
493 err_crypto_info:
494         memset(crypto_info, 0, sizeof(*crypto_info));
495 out:
496         return rc;
497 }
498
499 static int do_tls_setsockopt(struct sock *sk, int optname,
500                              char __user *optval, unsigned int optlen)
501 {
502         int rc = 0;
503
504         switch (optname) {
505         case TLS_TX:
506         case TLS_RX:
507                 lock_sock(sk);
508                 rc = do_tls_setsockopt_conf(sk, optval, optlen,
509                                             optname == TLS_TX);
510                 release_sock(sk);
511                 break;
512         default:
513                 rc = -ENOPROTOOPT;
514                 break;
515         }
516         return rc;
517 }
518
519 static int tls_setsockopt(struct sock *sk, int level, int optname,
520                           char __user *optval, unsigned int optlen)
521 {
522         struct tls_context *ctx = tls_get_ctx(sk);
523
524         if (level != SOL_TLS)
525                 return ctx->setsockopt(sk, level, optname, optval, optlen);
526
527         return do_tls_setsockopt(sk, optname, optval, optlen);
528 }
529
530 static struct tls_context *create_ctx(struct sock *sk)
531 {
532         struct inet_connection_sock *icsk = inet_csk(sk);
533         struct tls_context *ctx;
534
535         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
536         if (!ctx)
537                 return NULL;
538
539         icsk->icsk_ulp_data = ctx;
540         return ctx;
541 }
542
543 static int tls_hw_prot(struct sock *sk)
544 {
545         struct tls_context *ctx;
546         struct tls_device *dev;
547         int rc = 0;
548
549         mutex_lock(&device_mutex);
550         list_for_each_entry(dev, &device_list, dev_list) {
551                 if (dev->feature && dev->feature(dev)) {
552                         ctx = create_ctx(sk);
553                         if (!ctx)
554                                 goto out;
555
556                         ctx->hash = sk->sk_prot->hash;
557                         ctx->unhash = sk->sk_prot->unhash;
558                         ctx->sk_proto_close = sk->sk_prot->close;
559                         ctx->rx_conf = TLS_HW_RECORD;
560                         ctx->tx_conf = TLS_HW_RECORD;
561                         update_sk_prot(sk, ctx);
562                         rc = 1;
563                         break;
564                 }
565         }
566 out:
567         mutex_unlock(&device_mutex);
568         return rc;
569 }
570
571 static void tls_hw_unhash(struct sock *sk)
572 {
573         struct tls_context *ctx = tls_get_ctx(sk);
574         struct tls_device *dev;
575
576         mutex_lock(&device_mutex);
577         list_for_each_entry(dev, &device_list, dev_list) {
578                 if (dev->unhash)
579                         dev->unhash(dev, sk);
580         }
581         mutex_unlock(&device_mutex);
582         ctx->unhash(sk);
583 }
584
585 static int tls_hw_hash(struct sock *sk)
586 {
587         struct tls_context *ctx = tls_get_ctx(sk);
588         struct tls_device *dev;
589         int err;
590
591         err = ctx->hash(sk);
592         mutex_lock(&device_mutex);
593         list_for_each_entry(dev, &device_list, dev_list) {
594                 if (dev->hash)
595                         err |= dev->hash(dev, sk);
596         }
597         mutex_unlock(&device_mutex);
598
599         if (err)
600                 tls_hw_unhash(sk);
601         return err;
602 }
603
604 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
605                          struct proto *base)
606 {
607         prot[TLS_BASE][TLS_BASE] = *base;
608         prot[TLS_BASE][TLS_BASE].setsockopt     = tls_setsockopt;
609         prot[TLS_BASE][TLS_BASE].getsockopt     = tls_getsockopt;
610         prot[TLS_BASE][TLS_BASE].close          = tls_sk_proto_close;
611
612         prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
613         prot[TLS_SW][TLS_BASE].sendmsg          = tls_sw_sendmsg;
614         prot[TLS_SW][TLS_BASE].sendpage         = tls_sw_sendpage;
615
616         prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
617         prot[TLS_BASE][TLS_SW].recvmsg          = tls_sw_recvmsg;
618         prot[TLS_BASE][TLS_SW].close            = tls_sk_proto_close;
619
620         prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
621         prot[TLS_SW][TLS_SW].recvmsg    = tls_sw_recvmsg;
622         prot[TLS_SW][TLS_SW].close      = tls_sk_proto_close;
623
624 #ifdef CONFIG_TLS_DEVICE
625         prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
626         prot[TLS_HW][TLS_BASE].sendmsg          = tls_device_sendmsg;
627         prot[TLS_HW][TLS_BASE].sendpage         = tls_device_sendpage;
628
629         prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
630         prot[TLS_HW][TLS_SW].sendmsg            = tls_device_sendmsg;
631         prot[TLS_HW][TLS_SW].sendpage           = tls_device_sendpage;
632 #endif
633
634         prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
635         prot[TLS_HW_RECORD][TLS_HW_RECORD].hash         = tls_hw_hash;
636         prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash       = tls_hw_unhash;
637         prot[TLS_HW_RECORD][TLS_HW_RECORD].close        = tls_sk_proto_close;
638 }
639
640 static int tls_init(struct sock *sk)
641 {
642         int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
643         struct tls_context *ctx;
644         int rc = 0;
645
646         if (tls_hw_prot(sk))
647                 goto out;
648
649         /* The TLS ulp is currently supported only for TCP sockets
650          * in ESTABLISHED state.
651          * Supporting sockets in LISTEN state will require us
652          * to modify the accept implementation to clone rather then
653          * share the ulp context.
654          */
655         if (sk->sk_state != TCP_ESTABLISHED)
656                 return -ENOTSUPP;
657
658         /* allocate tls context */
659         ctx = create_ctx(sk);
660         if (!ctx) {
661                 rc = -ENOMEM;
662                 goto out;
663         }
664         ctx->setsockopt = sk->sk_prot->setsockopt;
665         ctx->getsockopt = sk->sk_prot->getsockopt;
666         ctx->sk_proto_close = sk->sk_prot->close;
667
668         /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
669         if (ip_ver == TLSV6 &&
670             unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) {
671                 mutex_lock(&tcpv6_prot_mutex);
672                 if (likely(sk->sk_prot != saved_tcpv6_prot)) {
673                         build_protos(tls_prots[TLSV6], sk->sk_prot);
674                         smp_store_release(&saved_tcpv6_prot, sk->sk_prot);
675                 }
676                 mutex_unlock(&tcpv6_prot_mutex);
677         }
678
679         ctx->tx_conf = TLS_BASE;
680         ctx->rx_conf = TLS_BASE;
681         update_sk_prot(sk, ctx);
682 out:
683         return rc;
684 }
685
686 void tls_register_device(struct tls_device *device)
687 {
688         mutex_lock(&device_mutex);
689         list_add_tail(&device->dev_list, &device_list);
690         mutex_unlock(&device_mutex);
691 }
692 EXPORT_SYMBOL(tls_register_device);
693
694 void tls_unregister_device(struct tls_device *device)
695 {
696         mutex_lock(&device_mutex);
697         list_del(&device->dev_list);
698         mutex_unlock(&device_mutex);
699 }
700 EXPORT_SYMBOL(tls_unregister_device);
701
702 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
703         .name                   = "tls",
704         .uid                    = TCP_ULP_TLS,
705         .user_visible           = true,
706         .owner                  = THIS_MODULE,
707         .init                   = tls_init,
708 };
709
710 static int __init tls_register(void)
711 {
712         build_protos(tls_prots[TLSV4], &tcp_prot);
713
714         tls_sw_proto_ops = inet_stream_ops;
715         tls_sw_proto_ops.poll = tls_sw_poll;
716         tls_sw_proto_ops.splice_read = tls_sw_splice_read;
717
718 #ifdef CONFIG_TLS_DEVICE
719         tls_device_init();
720 #endif
721         tcp_register_ulp(&tcp_tls_ulp_ops);
722
723         return 0;
724 }
725
726 static void __exit tls_unregister(void)
727 {
728         tcp_unregister_ulp(&tcp_tls_ulp_ops);
729 #ifdef CONFIG_TLS_DEVICE
730         tls_device_cleanup();
731 #endif
732 }
733
734 module_init(tls_register);
735 module_exit(tls_unregister);