diff --git a/.gitignore b/.gitignore index 8f4ce963..cb1e5d94 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ cscope.out *.d *.dwo *.ko +*.mod *.mod.c *.a Module.symvers diff --git a/src/Kbuild b/src/Kbuild index a0d433f2..8772824d 100644 --- a/src/Kbuild +++ b/src/Kbuild @@ -10,7 +10,18 @@ ccflags-$(if $(WIREGUARD_VERSION),y,) += -D'WIREGUARD_VERSION="$(WIREGUARD_VERSI wireguard-y := main.o noise.o device.o peer.o timers.o queueing.o send.o receive.o socket.o peerlookup.o allowedips.o ratelimiter.o cookie.o netlink.o +ifndef WOLFCRYPT include $(src)/crypto/Kbuild.include +endif include $(src)/compat/Kbuild.include obj-$(if $(KBUILD_EXTMOD),m,$(CONFIG_WIREGUARD)) := wireguard.o + +ifdef WOLFCRYPT +ifndef WOLFSSL_ROOT +WOLFSSL_ROOT = $(src)/../../../../wolfssl +endif +ccflags-y += -DUSE_WOLFCRYPT -I$(shell $(CC) -print-file-name=include) -I$(WOLFSSL_ROOT) -include $(src)/wolfcrypto_shim.h +wireguard-y += wolfcrypto_shim.o +KBUILD_EXTRA_SYMBOLS := $(WOLFSSL_ROOT)/linuxkm/Module.symvers +endif diff --git a/src/Makefile b/src/Makefile index b35ddbbc..2437fff2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -32,7 +32,11 @@ clean: @$(MAKE) -C $(KERNELDIR) M=$(PWD) clean module-install: +ifdef WOLFCRYPT + @$(MAKE) -C $(KERNELDIR) M=$(PWD) WIREGUARD_VERSION="$(WIREGUARD_VERSION)" INSTALL_MOD_DIR=wolfssl modules_install +else @$(MAKE) -C $(KERNELDIR) M=$(PWD) WIREGUARD_VERSION="$(WIREGUARD_VERSION)" modules_install +endif $(DEPMOD) -b "$(DEPMODBASEDIR)" -a $(KERNELRELEASE) install: module-install diff --git a/src/compat/Kbuild.include b/src/compat/Kbuild.include index e9376712..d5bdaf4d 100644 --- a/src/compat/Kbuild.include +++ b/src/compat/Kbuild.include @@ -2,7 +2,8 @@ # # Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. -kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) +#kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) +kbuild-dir := $(src) ccflags-y += -include $(kbuild-dir)/compat/compat.h asflags-y += -include $(kbuild-dir)/compat/compat-asm.h @@ -42,7 +43,7 @@ ccflags-y += -I$(kbuild-dir)/compat/udp_tunnel/include wireguard-y += compat/udp_tunnel/udp_tunnel.o endif -ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h"),) +ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h" "$(srctree)/include/crypto/utils.h"),) ccflags-y += -include $(kbuild-dir)/compat/memneq/include.h wireguard-y += compat/memneq/memneq.o endif diff --git a/src/compat/compat.h b/src/compat/compat.h index 9f6e725d..91492027 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -42,9 +42,9 @@ #error "WireGuard requires Linux >= 3.10" #endif -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -#error "WireGuard has been merged into Linux >= 5.6 and therefore this compatibility module is no longer required." -#endif +/* #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) */ +/* #error "WireGuard has been merged into Linux >= 5.6 and therefore this compatibility module is no longer required." */ +/* #endif */ #if defined(ISRHEL7) #include @@ -872,7 +872,7 @@ static inline void skb_mark_not_on_list(struct sk_buff *skb) #endif #endif -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) +#if !defined(USE_WOLFCRYPT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0)) #define blake2s_init zinc_blake2s_init #define blake2s_init_key zinc_blake2s_init_key #define blake2s_update zinc_blake2s_update @@ -1043,7 +1043,8 @@ static inline void skb_reset_redirect(struct sk_buff *skb) #define pre_exit exit #endif -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) +/* note, somewhere >5.4.0 and <=5.4.72, ip_tunnel_parse_protocol() was backported to the 5.4-LTS kernel. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) && ! (LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 72))) #include #include #include diff --git a/src/cookie.c b/src/cookie.c index 8b7d1fe0..9e88a7d6 100644 --- a/src/cookie.c +++ b/src/cookie.c @@ -10,8 +10,10 @@ #include "ratelimiter.h" #include "timers.h" +#ifndef WOLFCRYPTO_SHIM_H #include #include +#endif #include #include diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include index f2a312e9..2a5db4e9 100644 --- a/src/crypto/Kbuild.include +++ b/src/crypto/Kbuild.include @@ -45,7 +45,8 @@ kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) # Old kernels don't set this, which causes trouble. -.SECONDARY: +# breaks "make clean" on kernel 6.4 with ".NOTINTERMEDIATE and .SECONDARY are mutually exclusive." +# .SECONDARY: wireguard-y += $(addprefix crypto/zinc/,$(zinc-y)) ccflags-y += -I$(kbuild-dir)/crypto/include diff --git a/src/device.c b/src/device.c index 1155783a..bbd0ee74 100644 --- a/src/device.c +++ b/src/device.c @@ -11,6 +11,11 @@ #include "peer.h" #include "messages.h" +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) + /* kludge to work around broken pagemap.h, re min() and max(), due to 84429b675bcfd */ + #define _LINUX_PAGEMAP_H +#endif /* >= 6.12.0 */ + #include #include #include @@ -23,6 +28,16 @@ #include #include #include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 10)) || \ + (defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) + /* see linux 326534e837 and 8fa7292fee */ + #define del_timer timer_delete + #define del_timer_sync timer_delete_sync +#endif static LIST_HEAD(device_list); @@ -219,6 +234,15 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) return ret; } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) +static void ip_tunnel_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *tot) +{ + netdev_stats_to_stats64(tot, &dev->stats); + dev_fetch_sw_netstats(tot, dev->tstats); +} +#endif + static const struct net_device_ops netdev_ops = { .ndo_open = wg_open, .ndo_stop = wg_stop, @@ -282,7 +306,11 @@ static void wg_setup(struct net_device *dev) #else dev->tx_queue_len = 0; #endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) + dev->lltx = true; +#else dev->features |= NETIF_F_LLTX; +#endif dev->features |= WG_NETDEV_FEATURES; dev->hw_features |= WG_NETDEV_FEATURES; dev->hw_enc_features |= WG_NETDEV_FEATURES; @@ -300,14 +328,27 @@ static void wg_setup(struct net_device *dev) wg->dev = dev; } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) +static int wg_newlink(struct net_device *dev, + struct rtnl_newlink_params *params, + struct netlink_ext_ack *extack) +#else static int wg_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) +#endif { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) + struct net *link_net = rtnl_newlink_link_net(params); +#endif struct wg_device *wg = netdev_priv(dev); int ret = -ENOMEM; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) + rcu_assign_pointer(wg->creating_net, link_net); +#else rcu_assign_pointer(wg->creating_net, src_net); +#endif init_rwsem(&wg->static_identity.lock); mutex_init(&wg->socket_update_lock); mutex_init(&wg->device_update_lock); diff --git a/src/main.c b/src/main.c index 54350115..860e8925 100644 --- a/src/main.c +++ b/src/main.c @@ -14,17 +14,19 @@ #include #include -#include +#include #include static int __init mod_init(void) { int ret; +#ifndef WOLFCRYPTO_SHIM_H if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) || (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) || (ret = curve25519_mod_init())) return ret; +#endif #ifdef DEBUG if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || @@ -67,3 +69,11 @@ MODULE_VERSION(WIREGUARD_VERSION); MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); MODULE_INFO(intree, "Y"); + +#if defined(WOLFCRYPTO_SHIM_H) && defined(MODULE_IMPORT_NS) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 13, 0)) +MODULE_IMPORT_NS("WOLFSSL"); +#else +MODULE_IMPORT_NS(WOLFSSL); +#endif +#endif diff --git a/src/messages.h b/src/messages.h index 1d1ed18f..811617cf 100644 --- a/src/messages.h +++ b/src/messages.h @@ -6,9 +6,11 @@ #ifndef _WG_MESSAGES_H #define _WG_MESSAGES_H +#ifndef WOLFCRYPTO_SHIM_H #include #include #include +#endif #include #include diff --git a/src/netlink.c b/src/netlink.c index ef239ab1..ca3f91f5 100644 --- a/src/netlink.c +++ b/src/netlink.c @@ -198,7 +198,12 @@ static int wg_get_device_start(struct netlink_callback *cb) { struct wg_device *wg; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)) || \ + (defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) + wg = lookup_interface(genl_info_dump(cb)->attrs, cb->skb); +#else wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); +#endif if (IS_ERR(wg)) return PTR_ERR(wg); DUMP_CTX(cb)->wg = wg; diff --git a/src/peer.c b/src/peer.c index b3b6370e..430d26b1 100644 --- a/src/peer.c +++ b/src/peer.c @@ -58,8 +58,14 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, skb_queue_head_init(&peer->staged_packet_queue); wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || \ + (defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) + netif_napi_add_weight(wg->dev, &peer->napi, wg_packet_rx_poll, + NAPI_POLL_WEIGHT); +#else netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, NAPI_POLL_WEIGHT); +#endif napi_enable(&peer->napi); list_add_tail(&peer->peer_list, &wg->peer_list); INIT_LIST_HEAD(&peer->allowedips_list); diff --git a/src/queueing.h b/src/queueing.h index bab170b9..3991e5d3 100644 --- a/src/queueing.h +++ b/src/queueing.h @@ -80,9 +80,17 @@ static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) u8 sw_hash = skb->sw_hash; skb_scrub_packet(skb, true); + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0)) || \ + (defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) + memset(&skb->headers, 0, + sizeof skb->headers); +#else memset(&skb->headers_start, 0, offsetof(struct sk_buff, headers_end) - offsetof(struct sk_buff, headers_start)); +#endif + skb->pfmemalloc = pfmemalloc; if (encapsulating) { skb->hash = hash; diff --git a/src/receive.c b/src/receive.c index 172ef823..16d92c57 100644 --- a/src/receive.c +++ b/src/receive.c @@ -24,8 +24,14 @@ static void update_rx_stats(struct wg_peer *peer, size_t len) get_cpu_ptr(peer->device->dev->tstats); u64_stats_update_begin(&tstats->syncp); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)) || \ + (defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) + u64_stats_inc(&tstats->rx_packets); + u64_stats_add(&tstats->rx_bytes, len); +#else ++tstats->rx_packets; tstats->rx_bytes += len; +#endif peer->rx_bytes += len; u64_stats_update_end(&tstats->syncp); put_cpu_ptr(tstats); diff --git a/src/socket.c b/src/socket.c index c33e2c81..44051b89 100644 --- a/src/socket.c +++ b/src/socket.c @@ -49,7 +49,11 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, rt = dst_cache_get_ip4(cache, &fl.saddr); if (!rt) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) + security_sk_classify_flow(sock, flowi4_to_flowi_common(&fl)); +#else security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); +#endif if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { endpoint->src4.s_addr = 0; @@ -82,9 +86,15 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, } skb->ignore_df = 1; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 17, 0) udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, fl.fl4_dport, false, false); +#else + udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, false, 0); +#endif goto out; err: @@ -129,14 +139,24 @@ static int send6(struct wg_device *wg, struct sk_buff *skb, dst = dst_cache_get_ip6(cache, &fl.saddr); if (!dst) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) + security_sk_classify_flow(sock, flowi6_to_flowi_common(&fl)); +#else security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); +#endif if (unlikely(!ipv6_addr_any(&fl.saddr) && !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { endpoint->src6 = fl.saddr = in6addr_any; if (cache) dst_cache_reset(cache); } - dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, + /* 343d60aada (v4.16) adds net arg to ipv6_stub_impl.ipv6_dst_lookup, + * c4e85f73af (v5.5) adds it to ip6_dst_lookup_flow() (sunrise for the + * below code pattern), and ipv6_stub is slated for removal (along + * with net/ipv6_stubs.h) circa v7.1, linux-next commit + * 964870b4b9. + */ + dst = ip6_dst_lookup_flow(sock_net(sock), sock, &fl, NULL); if (unlikely(IS_ERR(dst))) { ret = PTR_ERR(dst); @@ -149,9 +169,15 @@ static int send6(struct wg_device *wg, struct sk_buff *skb, } skb->ignore_df = 1; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 17, 0) udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, ip6_dst_hoplimit(dst), 0, fl.fl6_sport, fl.fl6_dport, false); +#else + udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, false, 0); +#endif goto out; err: @@ -335,7 +361,11 @@ static void sock_free(struct sock *sock) if (unlikely(!sock)) return; sk_clear_memalloc(sock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 2, 0) udp_tunnel_sock_release(sock->sk_socket); +#else + udp_tunnel_sock_release(sock); +#endif } static void set_sock_opts(struct socket *sock) @@ -389,14 +419,22 @@ int wg_socket_init(struct wg_device *wg, u16 port) goto out; } set_sock_opts(new4); +#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 2, 0) setup_udp_tunnel_sock(net, new4, &cfg); +#else + setup_udp_tunnel_sock(net, new4->sk, &cfg); +#endif #if IS_ENABLED(CONFIG_IPV6) if (ipv6_mod_enabled()) { port6.local_udp_port = inet_sk(new4->sk)->inet_sport; ret = udp_sock_create(net, &port6, &new6); if (ret < 0) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 2, 0) udp_tunnel_sock_release(new4); +#else + udp_tunnel_sock_release(new4->sk); +#endif if (ret == -EADDRINUSE && !port && retries++ < 100) goto retry; pr_err("%s: Could not create IPv6 socket\n", @@ -404,7 +442,11 @@ int wg_socket_init(struct wg_device *wg, u16 port) goto out; } set_sock_opts(new6); +#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 2, 0) setup_udp_tunnel_sock(net, new6, &cfg); +#else + setup_udp_tunnel_sock(net, new6->sk, &cfg); +#endif } #endif diff --git a/src/timers.c b/src/timers.c index d54d32ac..24986e8c 100644 --- a/src/timers.c +++ b/src/timers.c @@ -27,6 +27,21 @@ * specified seconds. */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)) + #define get_random_u32_below prandom_u32_max +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 15, 0) + /* see linux 326534e837 and 8fa7292fee */ + #define timer_delete del_timer + #define timer_delete_sync del_timer_sync +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 16, 0) + /* see linux 41cb08555c and be54f8c558 */ + #define timer_container_of from_timer +#endif + static inline void mod_peer_timer(struct wg_peer *peer, struct timer_list *timer, unsigned long expires) @@ -40,15 +55,15 @@ static inline void mod_peer_timer(struct wg_peer *peer, static void wg_expired_retransmit_handshake(struct timer_list *timer) { - struct wg_peer *peer = from_timer(peer, timer, - timer_retransmit_handshake); + struct wg_peer *peer = timer_container_of(peer, timer, + timer_retransmit_handshake); if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); + &peer->endpoint.addr, (int)MAX_TIMER_HANDSHAKES + 2); - del_timer(&peer->timer_send_keepalive); + timer_delete(&peer->timer_send_keepalive); /* We drop all packets without a keypair and don't try again, * if we try unsuccessfully for too long to make a handshake. */ @@ -64,7 +79,7 @@ static void wg_expired_retransmit_handshake(struct timer_list *timer) ++peer->timer_handshake_attempts; pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, REKEY_TIMEOUT, + &peer->endpoint.addr, (int)REKEY_TIMEOUT, peer->timer_handshake_attempts + 1); /* We clear the endpoint address src address, in case this is @@ -78,7 +93,8 @@ static void wg_expired_retransmit_handshake(struct timer_list *timer) static void wg_expired_send_keepalive(struct timer_list *timer) { - struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); + struct wg_peer *peer = timer_container_of(peer, timer, + timer_send_keepalive); wg_packet_send_keepalive(peer); if (peer->timer_need_another_keepalive) { @@ -90,11 +106,12 @@ static void wg_expired_send_keepalive(struct timer_list *timer) static void wg_expired_new_handshake(struct timer_list *timer) { - struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); + struct wg_peer *peer = timer_container_of(peer, timer, + timer_new_handshake); pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + &peer->endpoint.addr, (int)(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT)); /* We clear the endpoint address src address, in case this is the cause * of trouble. */ @@ -104,7 +121,8 @@ static void wg_expired_new_handshake(struct timer_list *timer) static void wg_expired_zero_key_material(struct timer_list *timer) { - struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); + struct wg_peer *peer = timer_container_of(peer, timer, + timer_zero_key_material); rcu_read_lock_bh(); if (!READ_ONCE(peer->is_dead)) { @@ -126,7 +144,7 @@ static void wg_queued_expired_zero_key_material(struct work_struct *work) pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, REJECT_AFTER_TIME * 3); + &peer->endpoint.addr, (int)REJECT_AFTER_TIME * 3); wg_noise_handshake_clear(&peer->handshake); wg_noise_keypairs_clear(&peer->keypairs); wg_peer_put(peer); @@ -134,8 +152,8 @@ static void wg_queued_expired_zero_key_material(struct work_struct *work) static void wg_expired_send_persistent_keepalive(struct timer_list *timer) { - struct wg_peer *peer = from_timer(peer, timer, - timer_persistent_keepalive); + struct wg_peer *peer = timer_container_of(peer, timer, + timer_persistent_keepalive); if (likely(peer->persistent_keepalive_interval)) wg_packet_send_keepalive(peer); @@ -147,7 +165,7 @@ void wg_timers_data_sent(struct wg_peer *peer) if (!timer_pending(&peer->timer_new_handshake)) mod_peer_timer(peer, &peer->timer_new_handshake, jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + - prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); + get_random_u32_below(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); } /* Should be called after an authenticated data packet is received. */ @@ -167,7 +185,7 @@ void wg_timers_data_received(struct wg_peer *peer) */ void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) { - del_timer(&peer->timer_send_keepalive); + timer_delete(&peer->timer_send_keepalive); } /* Should be called after any type of authenticated packet is received, whether @@ -175,7 +193,7 @@ void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) */ void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) { - del_timer(&peer->timer_new_handshake); + timer_delete(&peer->timer_new_handshake); } /* Should be called after a handshake initiation message is sent. */ @@ -183,7 +201,7 @@ void wg_timers_handshake_initiated(struct wg_peer *peer) { mod_peer_timer(peer, &peer->timer_retransmit_handshake, jiffies + REKEY_TIMEOUT * HZ + - prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); + get_random_u32_below(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); } /* Should be called after a handshake response message is received and processed @@ -191,7 +209,7 @@ void wg_timers_handshake_initiated(struct wg_peer *peer) */ void wg_timers_handshake_complete(struct wg_peer *peer) { - del_timer(&peer->timer_retransmit_handshake); + timer_delete(&peer->timer_retransmit_handshake); peer->timer_handshake_attempts = 0; peer->sent_lastminute_handshake = false; ktime_get_real_ts64(&peer->walltime_last_handshake); @@ -234,10 +252,10 @@ void wg_timers_init(struct wg_peer *peer) void wg_timers_stop(struct wg_peer *peer) { - del_timer_sync(&peer->timer_retransmit_handshake); - del_timer_sync(&peer->timer_send_keepalive); - del_timer_sync(&peer->timer_new_handshake); - del_timer_sync(&peer->timer_zero_key_material); - del_timer_sync(&peer->timer_persistent_keepalive); + timer_delete_sync(&peer->timer_retransmit_handshake); + timer_delete_sync(&peer->timer_send_keepalive); + timer_delete_sync(&peer->timer_new_handshake); + timer_delete_sync(&peer->timer_zero_key_material); + timer_delete_sync(&peer->timer_persistent_keepalive); flush_work(&peer->clear_peer_work); } diff --git a/src/wolfcrypto_shim.c b/src/wolfcrypto_shim.c new file mode 100644 index 00000000..805494d4 --- /dev/null +++ b/src/wolfcrypto_shim.c @@ -0,0 +1,221 @@ +#include "wolfcrypto_shim.h" +#include + +int curve25519_generate_public( + uint8_t pub[static CURVE25519_KEY_SIZE], + const uint8_t secret[static CURVE25519_KEY_SIZE]) +{ + /* pubkey_main() calls curve25519_generate_public() with pub == secret, + * which doesn't work for wc_curve25519_make_pub(). + */ + uint8_t secret_copy[CURVE25519_KEY_SIZE]; + XMEMCPY(secret_copy, secret, CURVE25519_KEY_SIZE); + return !DBG_PRNT_NZ(wc_curve25519_make_pub(CURVE25519_KEY_SIZE, pub, CURVE25519_KEY_SIZE, secret_copy)); +} + +int curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]) { + WC_RNG *gRng = wc_rng_new(NULL /* nonce */, 0 /* nonceSz */, NULL /*heap */); + if (gRng) { + (void)DBG_PRNT_NZ(wc_curve25519_make_priv(gRng, (int)CURVE25519_KEY_SIZE, (byte *)secret)); + wc_rng_free(gRng); + return 0; + } else + return -ENOMEM; +} + +int blake2s(byte *out, const void *in, const void *key, const byte outlen, + const word32 inlen, byte keylen) +{ + Blake2s state; + + if ((in == NULL) || (out == NULL)) + return -1; + + if (DBG_PRNT_NZ(wc_InitBlake2s_WithKey(&state, (word32)outlen, (const byte *)key, (word32)keylen)) < 0) + return -1; + if (DBG_PRNT_NZ(wc_Blake2sUpdate(&state, (byte *)in, inlen)) < 0) + return -1; + return DBG_PRNT_NZ(wc_Blake2sFinal(&state, out, (word32)outlen)); +} + +void blake2s_hmac(byte *out, const byte *in, const byte *key, size_t outlen, size_t inlen, size_t keylen) { + Blake2s state; + word32 x_key[BLAKE2S_BLOCK_SIZE / sizeof(word32)]; + word32 i_hash[BLAKE2S_HASH_SIZE / sizeof(word32)]; + int i; + + if (outlen != BLAKE2S_HASH_SIZE) + return; + + if (keylen > BLAKE2S_BLOCK_SIZE) { + DBG_PRNT_NZ(wc_InitBlake2s(&state, BLAKE2S_HASH_SIZE)); + DBG_PRNT_NZ(wc_Blake2sUpdate(&state, key, keylen)); + DBG_PRNT_NZ(wc_Blake2sFinal(&state, (byte *)x_key, 0)); + } else { + XMEMCPY(x_key, key, keylen); + XMEMSET((byte *)x_key + keylen, 0, BLAKE2S_BLOCK_SIZE - keylen); + } + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + ((byte *)x_key)[i] ^= 0x36; + + DBG_PRNT_NZ(wc_InitBlake2s(&state, BLAKE2S_HASH_SIZE)); + DBG_PRNT_NZ(wc_Blake2sUpdate(&state, (byte *)x_key, BLAKE2S_BLOCK_SIZE)); + DBG_PRNT_NZ(wc_Blake2sUpdate(&state, in, inlen)); + DBG_PRNT_NZ(wc_Blake2sFinal(&state, (byte *)i_hash, 0)); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + ((byte *)x_key)[i] ^= 0x5c ^ 0x36; + + DBG_PRNT_NZ(wc_InitBlake2s(&state, BLAKE2S_HASH_SIZE)); + DBG_PRNT_NZ(wc_Blake2sUpdate(&state, (byte *)x_key, BLAKE2S_BLOCK_SIZE)); + DBG_PRNT_NZ(wc_Blake2sUpdate(&state, (byte *)i_hash, BLAKE2S_HASH_SIZE)); + DBG_PRNT_NZ(wc_Blake2sFinal(&state, (byte *)i_hash, 0)); + + XMEMCPY(out, i_hash, BLAKE2S_HASH_SIZE); + XMEMSET(x_key, 0, BLAKE2S_BLOCK_SIZE); + XMEMSET(i_hash, 0, BLAKE2S_HASH_SIZE); +} + +void blake2s256_hmac(byte *out, const byte *in, const byte *key, size_t inlen, size_t keylen) { + blake2s_hmac(out, in, key, BLAKE2S_HASH_SIZE, inlen, keylen); +} + +static bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, + const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + int isEncrypt) +{ + int ret = -1; + struct sg_mapping_iter miter; + unsigned int flags; + int sl; + ChaChaPoly_Aead *aead = (ChaChaPoly_Aead *)XMALLOC(sizeof *aead, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + if (WARN_ON(src_len > INT_MAX)) { + ret = BAD_FUNC_ARG; + goto out; + } + + { + byte IV[CHACHA20_POLY1305_AEAD_IV_SIZE] = {}; + memcpy(IV + 4, (byte *)&nonce, sizeof nonce); + if (WARN_ON(wc_ChaCha20Poly1305_Init(aead, key, IV, isEncrypt) < 0)) + goto out; + XMEMSET(IV, 0, sizeof IV); + } + + flags = SG_MITER_TO_SG; + if (!preemptible()) + flags |= SG_MITER_ATOMIC; + + sg_miter_start(&miter, src, sg_nents(src), flags); + + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + size_t length = min_t(size_t, sl, miter.length); + + if (! isEncrypt) { + if ((ret = wc_Poly1305Update(&aead->poly, miter.addr, length)) < 0) + goto out; + } + + if ((ret = wc_Chacha_Process(&aead->chacha, miter.addr, miter.addr, length)) < 0) + goto out; + + if (isEncrypt) { + if ((ret = wc_Poly1305Update(&aead->poly, miter.addr, length)) < 0) + goto out; + } + } + + if (aead->poly.leftover) { + if ((ret = wc_Poly1305_Pad(&aead->poly, (word32)aead->poly.leftover)) < 0) + goto out; + } + + if ((ret = wc_Poly1305_EncodeSizes(&aead->poly, ad_len, src_len)) < 0) + goto out; + + /* the remaining length (sl) really will be conditionally negative after + * iteration -- this is Jason Donenfeld's algorithm from + * chacha20poly1305_crypt_sg_inplace() in Linux + * lib/crypto/chacha20poly1305.c. + */ + if (sl <= -POLY1305_DIGEST_SIZE) { + if (isEncrypt) { + if ((ret = wc_Poly1305Final(&aead->poly, miter.addr + miter.length + sl)) < 0) + goto out; + } else { + byte outAuthTag[POLY1305_DIGEST_SIZE]; + + if ((ret = wc_Poly1305Final(&aead->poly, outAuthTag)) < 0) + goto out; + + if (ConstantCompare(outAuthTag, miter.addr + miter.length + sl, POLY1305_DIGEST_SIZE) != 0) { + ret = MAC_CMP_FAILED_E; + goto out; + } + } + } + + sg_miter_stop(&miter); + + if (sl > -POLY1305_DIGEST_SIZE) { + byte outAuthTag[POLY1305_DIGEST_SIZE]; + wc_Poly1305Final(&aead->poly, outAuthTag); + if (isEncrypt) { + scatterwalk_map_and_copy(outAuthTag, src, src_len, + sizeof outAuthTag, isEncrypt); + } else { + byte refAuthTag[POLY1305_DIGEST_SIZE]; + scatterwalk_map_and_copy(refAuthTag, src, src_len, + sizeof outAuthTag, isEncrypt); + if (ConstantCompare(outAuthTag, refAuthTag, POLY1305_DIGEST_SIZE) != 0) { + ret = MAC_CMP_FAILED_E; + goto out; + } + } + } + + ret = 0; + + out: + + if (aead) { + XMEMSET(&aead, 0, sizeof aead); + XFREE(aead, NULL, DYNAMIC_TYPE_TMP_BUFFER); + } + + (void)DBG_PRNT_NZ(ret); + + return ret == 0; +} + +bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + (void)simd_context; + return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, + nonce, key, 1); +} + +bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + (void)simd_context; + + if (unlikely(src_len < POLY1305_DIGEST_SIZE)) + return false; + + return chacha20poly1305_crypt_sg_inplace(src, + src_len - POLY1305_DIGEST_SIZE, + ad, ad_len, nonce, key, 0); +} diff --git a/src/wolfcrypto_shim.h b/src/wolfcrypto_shim.h new file mode 100644 index 00000000..0c84b816 --- /dev/null +++ b/src/wolfcrypto_shim.h @@ -0,0 +1,251 @@ +#ifndef WOLFCRYPTO_SHIM_H +#define WOLFCRYPTO_SHIM_H + +#include +#ifndef WOLFSSL_LINUXKM +#error libwolfssl configured without --enable-linuxkm +#endif +#ifndef HAVE_CURVE25519 +#error libwolfssl missing HAVE_CURVE25519 +#endif +#ifndef HAVE_BLAKE2S +#error libwolfssl missing HAVE_BLAKE2S +#endif +#ifndef HAVE_CHACHA +#error libwolfssl missing HAVE_CHACHA +#endif +#ifndef HAVE_POLY1305 +#error libwolfssl missing HAVE_POLY1305 +#endif + +/* internal file misc.c at commit d9f7629296 has inline CopyString() that calls + * XMALLOC(). + */ +#include +#include +#include +#define WOLFSSL_MISC_INCLUDED +#undef min +#undef max +#include + +#include +#define CURVE25519_KEY_SIZE CURVE25519_KEYSIZE + +#include + +#include +#define CHACHA20POLY1305_KEY_SIZE CHACHA20_POLY1305_AEAD_KEYSIZE +#define CHACHA20POLY1305_AUTHTAG_SIZE CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE +#define XCHACHA20POLY1305_NONCE_SIZE 24 /* CHACHA20_POLY1305_AEAD_IV_SIZE * 2 */ + +#include + +#include +#define BLAKE2S_HASH_SIZE BLAKE2S_256 +#define BLAKE2S_BLOCK_SIZE 64 + +#undef SHA256_BLOCK_SIZE +#undef SHA256_DIGEST_SIZE +#undef SHA224_BLOCK_SIZE +#undef SHA224_DIGEST_SIZE + +#include +#include +#include +#include + +#define PRNT_NZ(...) \ + ({ \ + int _ret = (__VA_ARGS__); \ + if (_ret) { \ + printk(KERN_NOTICE "%s@%d: %d\n", __FILE__, __LINE__, _ret); \ + } \ + _ret; \ + }) + +#ifdef DEBUG +#define DBG_PRNT_NZ(...) PRNT_NZ(__VA_ARGS__) +#else +#define DBG_PRNT_NZ(...) (__VA_ARGS__) +#endif + +struct blake2s_state { + Blake2s blake2s; +}; +#define blake2s_init(...) wc_wg_blake2s_init(__VA_ARGS__) +static inline __attribute__((unused)) void blake2s_init( + struct blake2s_state *state, + size_t outlen) +{ + DBG_PRNT_NZ(wc_InitBlake2s(&state->blake2s, (word32)outlen)); +} +#define blake2s_init_key(...) wc_wg_blake2s_init_key(__VA_ARGS__) +static inline __attribute__((unused)) void blake2s_init_key( + struct blake2s_state *state, + size_t outlen, + const void *key, + const size_t keylen) +{ + DBG_PRNT_NZ(wc_InitBlake2s_WithKey( + &state->blake2s, + (word32)outlen, + (const byte *)key, + (word32)keylen)); +} +#define blake2s_update(...) wc_wg_blake2s_update(__VA_ARGS__) +static inline __attribute__((unused)) void blake2s_update( + struct blake2s_state *state, + const u8 *in, + size_t inlen) +{ + DBG_PRNT_NZ(wc_Blake2sUpdate(&state->blake2s, (const byte *)in, (word32)inlen)); +} +#define blake2s_final(...) wc_wg_blake2s_final(__VA_ARGS__) +static inline __attribute__((unused)) void blake2s_final( + struct blake2s_state *state, + const u8 *out) { + DBG_PRNT_NZ(wc_Blake2sFinal(&state->blake2s, (byte *)out, 0)); +} + +#define blake2s(...) wc_wg_simple_blake2s(__VA_ARGS__) +extern int blake2s(byte *out, const void *in, const void *key, const byte outlen, + const word32 inlen, byte keylen); + +#define blake2s256_hmac(...) wc_wg_blake2s256_hmac(__VA_ARGS__) +extern void blake2s256_hmac( + byte *out, + const byte *in, + const byte *key, + size_t inlen, + size_t keylen); + +#define blake2s_hmac(...) wc_wg_blake2s_hmac(__VA_ARGS__) +extern void blake2s_hmac( + byte *out, + const byte *in, + const byte *key, + size_t outlen, + size_t inlen, + size_t keylen); + +#define curve25519_generate_public(...) curve25519_generate_public_wolfshim(__VA_ARGS__) +extern int curve25519_generate_public( + uint8_t pub[static CURVE25519_KEYSIZE], + const uint8_t secret[static CURVE25519_KEYSIZE]); + +#define curve25519_generate_secret(...) curve25519_generate_secret_wolfshim(__VA_ARGS__) +extern int curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]); + +#define curve25519_clamp_secret(...) curve25519_clamp_secret_wolfshim(__VA_ARGS__) +static inline void curve25519_clamp_secret(u8 key[CURVE25519_KEY_SIZE]) +{ + key[0] &= 248; + key[CURVE25519_KEY_SIZE-1] &= 63; /* same &=127 because |=64 after */ + key[CURVE25519_KEY_SIZE-1] |= 64; +} + +#define curve25519(...) curve25519_wolfshim(__VA_ARGS__) +static inline bool curve25519( + uint8_t mypublic[static CURVE25519_KEY_SIZE], + const uint8_t secret[static CURVE25519_KEY_SIZE], + const uint8_t basepoint[static CURVE25519_KEY_SIZE]) +{ + return (wc_curve25519_generic( + CURVE25519_KEY_SIZE, (byte *)mypublic, + CURVE25519_KEY_SIZE, (byte *)secret, + CURVE25519_KEY_SIZE, (byte *)basepoint) + == 0 ? true : false); +} + +static inline __attribute__((unused)) void +chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { + word64 inIV[2] = { 0, cpu_to_le64(nonce) }; + ChaChaPoly_Aead aead; + + if (DBG_PRNT_NZ(wc_ChaCha20Poly1305_Init(&aead, key, (const byte *)inIV, + CHACHA20_POLY1305_AEAD_ENCRYPT))) + return; + + DBG_PRNT_NZ(wc_ChaCha20Poly1305_UpdateAad(&aead, ad, (u32)ad_len)); + if (src_len) + DBG_PRNT_NZ(wc_ChaCha20Poly1305_UpdateData(&aead, src, dst, + (u32)src_len)); + PRNT_NZ(wc_ChaCha20Poly1305_Final(&aead, dst + src_len)); +} + +static inline __attribute__((unused)) bool +chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { + word64 inIV[2] = { 0, cpu_to_le64(nonce) }; + + int ret = 0; + ChaChaPoly_Aead aead; + byte calculatedAuthTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE] = {}; + + if (DBG_PRNT_NZ + (wc_ChaCha20Poly1305_Init( + &aead, + key, + (const u8 *)inIV + sizeof inIV - CHACHA20_POLY1305_AEAD_IV_SIZE, + CHACHA20_POLY1305_AEAD_DECRYPT))) + return false; + ret |= DBG_PRNT_NZ(wc_ChaCha20Poly1305_UpdateAad(&aead, ad, (u32)ad_len)); + if (dst) + ret |= DBG_PRNT_NZ(wc_ChaCha20Poly1305_UpdateData( + &aead, + src, + dst, + src_len - CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE)); + ret |= DBG_PRNT_NZ(wc_ChaCha20Poly1305_Final(&aead, calculatedAuthTag)); + ret |= DBG_PRNT_NZ(wc_ChaCha20Poly1305_CheckTag( + src + src_len - CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE, + calculatedAuthTag)); + + return ret == 0; +} + +#define xchacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key) \ + DBG_PRNT_NZ(wc_XChaCha20Poly1305_Encrypt( \ + dst, \ + (src_len) + POLY1305_DIGEST_SIZE, \ + src, src_len, \ + ad, ad_len, \ + nonce, XCHACHA20POLY1305_NONCE_SIZE, \ + key, CHACHA20POLY1305_KEY_SIZE)) + +#define xchacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce, key) \ + (DBG_PRNT_NZ(wc_XChaCha20Poly1305_Decrypt( \ + dst, \ + (src_len) - POLY1305_DIGEST_SIZE, \ + src, src_len, \ + ad, ad_len, \ + nonce, XCHACHA20POLY1305_NONCE_SIZE, \ + key, CHACHA20POLY1305_KEY_SIZE)) == 0) + +#define chacha20poly1305_encrypt_sg_inplace(...) \ + chacha20poly1305_encrypt_sg_inplace_wolfshim(__VA_ARGS__) +extern bool chacha20poly1305_encrypt_sg_inplace( \ + struct scatterlist *src, \ + size_t src_len, \ + const u8 *ad, const size_t ad_len, \ + const u64 nonce, \ + const u8 key[CHACHA20POLY1305_KEY_SIZE], \ + simd_context_t *simd_context); + +#define chacha20poly1305_decrypt_sg_inplace(...) \ + chacha20poly1305_decrypt_sg_inplace_wolfshim(__VA_ARGS__) +extern bool chacha20poly1305_decrypt_sg_inplace( \ + struct scatterlist *src, \ + size_t src_len, \ + const u8 *ad, const size_t ad_len, \ + const u64 nonce, \ + const u8 key[CHACHA20POLY1305_KEY_SIZE], \ + simd_context_t *simd_context); + +#endif