FD.io VPP  v19.08-27-gf4dcae4
Vector Packet Processing
esp_encrypt.c
Go to the documentation of this file.
1 /*
2  * esp_encrypt.c : IPSec ESP encrypt node
3  *
4  * Copyright (c) 2015 Cisco and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
21 #include <vnet/udp/udp.h>
22 
23 #include <vnet/crypto/crypto.h>
24 
25 #include <vnet/ipsec/ipsec.h>
26 #include <vnet/ipsec/esp.h>
27 
28 #define foreach_esp_encrypt_next \
29 _(DROP, "error-drop") \
30 _(IP4_LOOKUP, "ip4-lookup") \
31 _(IP6_LOOKUP, "ip6-lookup") \
32 _(INTERFACE_OUTPUT, "interface-output")
33 
34 #define _(v, s) ESP_ENCRYPT_NEXT_##v,
35 typedef enum
36 {
38 #undef _
41 
42 #define foreach_esp_encrypt_error \
43  _(RX_PKTS, "ESP pkts received") \
44  _(SEQ_CYCLED, "sequence number cycled (packet dropped)") \
45  _(CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)") \
46  _(CHAINED_BUFFER, "chained buffers (packet dropped)") \
47  _(NO_TRAILER_SPACE, "no trailer space (packet dropped)")
48 
49 typedef enum
50 {
51 #define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
53 #undef _
56 
57 static char *esp_encrypt_error_strings[] = {
58 #define _(sym,string) string,
60 #undef _
61 };
62 
63 typedef struct
64 {
73 
74 /* packet trace format function */
75 static u8 *
76 format_esp_encrypt_trace (u8 * s, va_list * args)
77 {
78  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
79  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
80  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
81 
82  s =
83  format (s,
84  "esp: sa-index %d spi %u (0x%08x) seq %u sa-seq-hi %u crypto %U integrity %U%s",
85  t->sa_index, t->spi, t->spi, t->seq, t->sa_seq_hi,
88  t->udp_encap ? " udp-encap-enabled" : "");
89  return s;
90 }
91 
92 /* pad packet in input buffer */
94 esp_add_footer_and_icv (vlib_buffer_t * b, u8 block_size, u8 icv_sz)
95 {
96  static const u8 pad_data[ESP_MAX_BLOCK_SIZE] = {
97  0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
98  0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x00, 0x00,
99  };
100 
101  u16 min_length = b->current_length + sizeof (esp_footer_t);
102  u16 new_length = round_pow2 (min_length, block_size);
103  u8 pad_bytes = new_length - min_length;
105  new_length - sizeof (esp_footer_t));
106 
107  if (pad_bytes)
108  clib_memcpy_fast ((u8 *) f - pad_bytes, pad_data, ESP_MAX_BLOCK_SIZE);
109 
110  f->pad_length = pad_bytes;
111  b->current_length = new_length + icv_sz;
112  return &f->next_header;
113 }
114 
116 esp_update_ip4_hdr (ip4_header_t * ip4, u16 len, int is_transport, int is_udp)
117 {
118  ip_csum_t sum;
119  u16 old_len;
120 
121  len = clib_net_to_host_u16 (len);
122  old_len = ip4->length;
123 
124  if (is_transport)
125  {
126  u8 prot = is_udp ? IP_PROTOCOL_UDP : IP_PROTOCOL_IPSEC_ESP;
127 
128  sum = ip_csum_update (ip4->checksum, ip4->protocol,
129  prot, ip4_header_t, protocol);
130  ip4->protocol = prot;
131 
132  sum = ip_csum_update (sum, old_len, len, ip4_header_t, length);
133  }
134  else
135  sum = ip_csum_update (ip4->checksum, old_len, len, ip4_header_t, length);
136 
137  ip4->length = len;
138  ip4->checksum = ip_csum_fold (sum);
139 }
140 
143 {
144  clib_memcpy_fast (udp, &sa->udp_hdr, sizeof (udp_header_t));
145  udp->length = clib_net_to_host_u16 (len);
146 }
147 
150 {
151 #ifdef CLIB_HAVE_VEC128
152  static const u8x16 ext_hdr_types = {
153  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS,
154  IP_PROTOCOL_IPV6_ROUTE,
155  IP_PROTOCOL_IPV6_FRAGMENTATION,
156  };
157 
158  return !u8x16_is_all_zero (ext_hdr_types == u8x16_splat (nexthdr));
159 #else
160  return ((nexthdr ^ IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) |
161  (nexthdr ^ IP_PROTOCOL_IPV6_ROUTE) |
162  (nexthdr ^ IP_PROTOCOL_IPV6_FRAGMENTATION) != 0);
163 #endif
164 }
165 
168 {
169  /* this code assumes that HbH, route and frag headers will be before
170  others, if that is not the case, they will end up encrypted */
171 
172  u8 len = sizeof (ip6_header_t);
173  ip6_ext_header_t *p;
174 
175  /* if next packet doesn't have ext header */
176  if (ext_hdr_is_pre_esp (ip6->protocol) == 0)
177  return len;
178 
179  p = (void *) (ip6 + 1);
180  len += ip6_ext_header_len (p);
181 
182  while (ext_hdr_is_pre_esp (p->next_hdr))
183  {
184  len += ip6_ext_header_len (p);
185  p = ip6_ext_next_header (p);
186  }
187 
188  return len;
189 }
190 
193  u16 * next, u16 buffer_data_size)
194 {
195  if (b->current_data + b->current_length <= buffer_data_size)
196  return 0;
197 
198  b->current_length -= buffer_data_size - b->current_data;
199  b->error = node->errors[ESP_ENCRYPT_ERROR_NO_TRAILER_SPACE];
200  next[0] = ESP_ENCRYPT_NEXT_DROP;
201  return 1;
202 }
203 
206  vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts)
207 {
208  u32 n_fail, n_ops = vec_len (ops);
209  vnet_crypto_op_t *op = ops;
210 
211  if (n_ops == 0)
212  return;
213 
214  n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
215 
216  while (n_fail)
217  {
218  ASSERT (op - ops < n_ops);
219 
220  if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
221  {
222  u32 bi = op->user_data;
223  b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR];
224  nexts[bi] = ESP_ENCRYPT_NEXT_DROP;
225  n_fail--;
226  }
227  op++;
228  }
229 }
230 
231 typedef struct
232 {
233  u32 salt;
234  u64 iv;
235 } __clib_packed esp_gcm_nonce_t;
236 
237 STATIC_ASSERT_SIZEOF (esp_gcm_nonce_t, 12);
238 
241  vlib_frame_t * frame, int is_ip6, int is_tun)
242 {
243  ipsec_main_t *im = &ipsec_main;
245  u32 *from = vlib_frame_vector_args (frame);
246  u32 n_left = frame->n_vectors;
247  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
248  u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
249  esp_gcm_nonce_t nonces[VLIB_FRAME_SIZE], *nonce = nonces;
250  u32 thread_index = vm->thread_index;
251  u16 buffer_data_size = vlib_buffer_get_default_data_size (vm);
252  u32 current_sa_index = ~0, current_sa_packets = 0;
253  u32 current_sa_bytes = 0, spi = 0;
254  u8 block_sz = 0, iv_sz = 0, icv_sz = 0;
255  ipsec_sa_t *sa0 = 0;
256 
257  vlib_get_buffers (vm, from, b, n_left);
260 
261  while (n_left > 0)
262  {
263  u32 sa_index0;
264  dpo_id_t *dpo;
265  esp_header_t *esp;
266  u8 *payload, *next_hdr_ptr;
267  u16 payload_len;
268  u32 hdr_len;
269 
270  if (n_left > 2)
271  {
272  u8 *p;
273  vlib_prefetch_buffer_header (b[2], LOAD);
274  p = vlib_buffer_get_current (b[1]);
278  }
279 
280  if (is_tun)
281  {
282  /* we are on a ipsec tunnel's feature arc */
283  u32 next0;
284  sa_index0 = *(u32 *) vnet_feature_next_with_data (&next0, b[0],
285  sizeof
286  (sa_index0));
287  next[0] = next0;
288  }
289  else
290  sa_index0 = vnet_buffer (b[0])->ipsec.sad_index;
291 
292  if (sa_index0 != current_sa_index)
293  {
294  if (current_sa_packets)
296  current_sa_index,
297  current_sa_packets,
298  current_sa_bytes);
299  current_sa_packets = current_sa_bytes = 0;
300 
301  sa0 = pool_elt_at_index (im->sad, sa_index0);
302  current_sa_index = sa_index0;
303  spi = clib_net_to_host_u32 (sa0->spi);
304  block_sz = sa0->crypto_block_size;
305  icv_sz = sa0->integ_icv_size;
306  iv_sz = sa0->crypto_iv_size;
307  }
308 
309  if (vlib_buffer_chain_linearize (vm, b[0]) != 1)
310  {
311  b[0]->error = node->errors[ESP_ENCRYPT_ERROR_CHAINED_BUFFER];
312  next[0] = ESP_ENCRYPT_NEXT_DROP;
313  goto trace;
314  }
315 
316  if (PREDICT_FALSE (esp_seq_advance (sa0)))
317  {
318  b[0]->error = node->errors[ESP_ENCRYPT_ERROR_SEQ_CYCLED];
319  next[0] = ESP_ENCRYPT_NEXT_DROP;
320  goto trace;
321  }
322 
323  /* space for IV */
324  hdr_len = iv_sz;
325 
326  if (ipsec_sa_is_set_IS_TUNNEL (sa0))
327  {
328  payload = vlib_buffer_get_current (b[0]);
329  next_hdr_ptr = esp_add_footer_and_icv (b[0], block_sz, icv_sz);
330  payload_len = b[0]->current_length;
331 
332  if (esp_trailer_icv_overflow (node, b[0], next, buffer_data_size))
333  goto trace;
334 
335  /* ESP header */
336  hdr_len += sizeof (*esp);
337  esp = (esp_header_t *) (payload - hdr_len);
338 
339  /* optional UDP header */
340  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
341  {
342  hdr_len += sizeof (udp_header_t);
343  esp_fill_udp_hdr (sa0, (udp_header_t *) (payload - hdr_len),
344  payload_len + hdr_len);
345  }
346 
347  /* IP header */
348  if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))
349  {
350  ip6_header_t *ip6;
351  u16 len = sizeof (ip6_header_t);
352  hdr_len += len;
353  ip6 = (ip6_header_t *) (payload - hdr_len);
354  clib_memcpy_fast (ip6, &sa0->ip6_hdr, len);
355  *next_hdr_ptr = (is_ip6 ?
356  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
357  len = payload_len + hdr_len - len;
358  ip6->payload_length = clib_net_to_host_u16 (len);
359  }
360  else
361  {
362  ip4_header_t *ip4;
363  u16 len = sizeof (ip4_header_t);
364  hdr_len += len;
365  ip4 = (ip4_header_t *) (payload - hdr_len);
366  clib_memcpy_fast (ip4, &sa0->ip4_hdr, len);
367  *next_hdr_ptr = (is_ip6 ?
368  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
369  len = payload_len + hdr_len;
370  esp_update_ip4_hdr (ip4, len, /* is_transport */ 0, 0);
371  }
372 
373  dpo = &sa0->dpo;
374  if (!is_tun)
375  {
376  next[0] = dpo->dpoi_next_node;
377  vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo->dpoi_index;
378  }
379  }
380  else /* transport mode */
381  {
382  u8 *l2_hdr, l2_len, *ip_hdr, ip_len;
383  udp_header_t *udp = 0;
384  u8 *old_ip_hdr = vlib_buffer_get_current (b[0]);
385 
386  ip_len = is_ip6 ?
387  esp_get_ip6_hdr_len ((ip6_header_t *) old_ip_hdr) :
388  ip4_header_bytes ((ip4_header_t *) old_ip_hdr);
389 
390  vlib_buffer_advance (b[0], ip_len);
391  payload = vlib_buffer_get_current (b[0]);
392  next_hdr_ptr = esp_add_footer_and_icv (b[0], block_sz, icv_sz);
393  payload_len = b[0]->current_length;
394 
395  if (esp_trailer_icv_overflow (node, b[0], next, buffer_data_size))
396  goto trace;
397 
398  /* ESP header */
399  hdr_len += sizeof (*esp);
400  esp = (esp_header_t *) (payload - hdr_len);
401 
402  /* optional UDP header */
403  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
404  {
405  hdr_len += sizeof (udp_header_t);
406  udp = (udp_header_t *) (payload - hdr_len);
407  }
408 
409  /* IP header */
410  hdr_len += ip_len;
411  ip_hdr = payload - hdr_len;
412 
413  /* L2 header */
414  if (!is_tun)
415  {
416  l2_len = vnet_buffer (b[0])->ip.save_rewrite_length;
417  hdr_len += l2_len;
418  l2_hdr = payload - hdr_len;
419 
420  /* copy l2 and ip header */
421  clib_memcpy_le32 (l2_hdr, old_ip_hdr - l2_len, l2_len);
422  }
423  else
424  l2_len = 0;
425 
426  clib_memcpy_le64 (ip_hdr, old_ip_hdr, ip_len);
427 
428  if (is_ip6)
429  {
430  ip6_header_t *ip6 = (ip6_header_t *) (ip_hdr);
431  *next_hdr_ptr = ip6->protocol;
432  ip6->protocol = IP_PROTOCOL_IPSEC_ESP;
433  ip6->payload_length =
434  clib_host_to_net_u16 (payload_len + hdr_len - l2_len -
435  ip_len);
436  }
437  else
438  {
439  u16 len;
440  ip4_header_t *ip4 = (ip4_header_t *) (ip_hdr);
441  *next_hdr_ptr = ip4->protocol;
442  len = payload_len + hdr_len - l2_len;
443  if (udp)
444  {
445  esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 1);
446  esp_fill_udp_hdr (sa0, udp, len - ip_len);
447  }
448  else
449  esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 0);
450  }
451 
452  if (!is_tun)
453  next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT;
454  }
455 
456  esp->spi = spi;
457  esp->seq = clib_net_to_host_u32 (sa0->seq);
458 
459  if (sa0->crypto_enc_op_id)
460  {
461  vnet_crypto_op_t *op;
464  op->src = op->dst = payload;
465  op->key_index = sa0->crypto_key_index;
466  op->len = payload_len - icv_sz;
467  op->user_data = b - bufs;
468 
469  if (ipsec_sa_is_set_IS_AEAD (sa0))
470  {
471  /*
472  * construct the AAD in a scratch space in front
473  * of the IP header.
474  */
475  op->aad = payload - hdr_len - sizeof (esp_aead_t);
476 
477  esp_aad_fill (op, esp, sa0);
478 
479  op->tag = payload + op->len;
480  op->tag_len = 16;
481 
482  u64 *iv = (u64 *) (payload - iv_sz);
483  nonce->salt = sa0->salt;
484  nonce->iv = *iv = clib_host_to_net_u64 (sa0->gcm_iv_counter++);
485  op->iv = (u8 *) nonce;
486  nonce++;
487  }
488  else
489  {
490  op->iv = payload - iv_sz;
492  }
493  }
494 
495  if (sa0->integ_op_id)
496  {
497  vnet_crypto_op_t *op;
499  vnet_crypto_op_init (op, sa0->integ_op_id);
500  op->src = payload - iv_sz - sizeof (esp_header_t);
501  op->digest = payload + payload_len - icv_sz;
502  op->key_index = sa0->integ_key_index;
503  op->digest_len = icv_sz;
504  op->len = payload_len - icv_sz + iv_sz + sizeof (esp_header_t);
505  op->user_data = b - bufs;
506  if (ipsec_sa_is_set_USE_ESN (sa0))
507  {
508  u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi);
509  clib_memcpy_fast (op->digest, &seq_hi, sizeof (seq_hi));
510  op->len += sizeof (seq_hi);
511  }
512  }
513 
514  vlib_buffer_advance (b[0], 0LL - hdr_len);
515 
516  current_sa_packets += 1;
517  current_sa_bytes += payload_len;
518 
519  trace:
520  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
521  {
522  esp_encrypt_trace_t *tr = vlib_add_trace (vm, node, b[0],
523  sizeof (*tr));
524  tr->sa_index = sa_index0;
525  tr->spi = sa0->spi;
526  tr->seq = sa0->seq;
527  tr->sa_seq_hi = sa0->seq_hi;
528  tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa0);
529  tr->crypto_alg = sa0->crypto_alg;
530  tr->integ_alg = sa0->integ_alg;
531  }
532  /* next */
533  n_left -= 1;
534  next += 1;
535  b += 1;
536  }
537 
539  current_sa_index, current_sa_packets,
540  current_sa_bytes);
541  esp_process_ops (vm, node, ptd->crypto_ops, bufs, nexts);
542  esp_process_ops (vm, node, ptd->integ_ops, bufs, nexts);
543 
545  ESP_ENCRYPT_ERROR_RX_PKTS, frame->n_vectors);
546 
547  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
548  return frame->n_vectors;
549 }
550 
552  vlib_node_runtime_t * node,
553  vlib_frame_t * from_frame)
554 {
555  return esp_encrypt_inline (vm, node, from_frame, 0 /* is_ip6 */ , 0);
556 }
557 
558 /* *INDENT-OFF* */
560  .name = "esp4-encrypt",
561  .vector_size = sizeof (u32),
562  .format_trace = format_esp_encrypt_trace,
564 
566  .error_strings = esp_encrypt_error_strings,
567 
568  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
569  .next_nodes = {
570 #define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n,
572 #undef _
573  },
574 };
575 /* *INDENT-ON* */
576 
578  vlib_node_runtime_t * node,
579  vlib_frame_t * from_frame)
580 {
581  return esp_encrypt_inline (vm, node, from_frame, 1 /* is_ip6 */ , 0);
582 }
583 
584 /* *INDENT-OFF* */
586  .name = "esp6-encrypt",
587  .vector_size = sizeof (u32),
588  .format_trace = format_esp_encrypt_trace,
590 
592  .error_strings = esp_encrypt_error_strings,
593 
594  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
595  .next_nodes = {
596 #define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n,
598 #undef _
599  },
600 };
601 /* *INDENT-ON* */
602 
604  vlib_node_runtime_t * node,
605  vlib_frame_t * from_frame)
606 {
607  return esp_encrypt_inline (vm, node, from_frame, 0 /* is_ip6 */ , 1);
608 }
609 
610 /* *INDENT-OFF* */
612  .name = "esp4-encrypt-tun",
613  .vector_size = sizeof (u32),
614  .format_trace = format_esp_encrypt_trace,
616 
618  .error_strings = esp_encrypt_error_strings,
619 
620  .n_next_nodes = 1,
621  .next_nodes = {
622  [ESP_ENCRYPT_NEXT_DROP] = "ip4-drop",
623  },
624 };
625 
626 VNET_FEATURE_INIT (esp4_encrypt_tun_feat_node, static) =
627 {
628  .arc_name = "ip4-output",
629  .node_name = "esp4-encrypt-tun",
630  .runs_before = VNET_FEATURES ("adj-midchain-tx"),
631 };
632 
633 VNET_FEATURE_INIT (esp4_ethernet_encrypt_tun_feat_node, static) =
634 {
635  .arc_name = "ethernet-output",
636  .node_name = "esp4-encrypt-tun",
637  .runs_before = VNET_FEATURES ("adj-midchain-tx", "adj-midchain-tx-no-count"),
638 };
639 /* *INDENT-ON* */
640 
642  vlib_node_runtime_t * node,
643  vlib_frame_t * from_frame)
644 {
645  return esp_encrypt_inline (vm, node, from_frame, 1 /* is_ip6 */ , 1);
646 }
647 
648 /* *INDENT-OFF* */
650  .name = "esp6-encrypt-tun",
651  .vector_size = sizeof (u32),
652  .format_trace = format_esp_encrypt_trace,
654 
656  .error_strings = esp_encrypt_error_strings,
657 
658  .n_next_nodes = 1,
659  .next_nodes = {
660  [ESP_ENCRYPT_NEXT_DROP] = "ip6-drop",
661  },
662 };
663 
664 VNET_FEATURE_INIT (esp6_encrypt_tun_feat_node, static) =
665 {
666  .arc_name = "ip6-output",
667  .node_name = "esp6-encrypt-tun",
668  .runs_before = VNET_FEATURES ("adj-midchain-tx"),
669 };
670 /* *INDENT-ON* */
671 
672 typedef struct
673 {
676 
677 static u8 *
678 format_esp_no_crypto_trace (u8 * s, va_list * args)
679 {
680  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
681  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
682  esp_no_crypto_trace_t *t = va_arg (*args, esp_no_crypto_trace_t *);
683 
684  s = format (s, "esp-no-crypto: sa-index %u", t->sa_index);
685 
686  return s;
687 }
688 
689 enum
690 {
693 };
694 
695 enum
696 {
698 };
699 
700 static char *esp_no_crypto_error_strings[] = {
701  "Outbound ESP packets received",
702 };
703 
706  vlib_frame_t * frame)
707 {
708  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
709  u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
710  u32 *from = vlib_frame_vector_args (frame);
711  u32 n_left = frame->n_vectors;
712 
713  vlib_get_buffers (vm, from, b, n_left);
714 
715  while (n_left > 0)
716  {
717  u32 next0;
718  u32 sa_index0;
719 
720  /* packets are always going to be dropped, but get the sa_index */
721  sa_index0 = *(u32 *) vnet_feature_next_with_data (&next0, b[0],
722  sizeof (sa_index0));
723 
724  next[0] = ESP_NO_CRYPTO_NEXT_DROP;
725 
726  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
727  {
728  esp_no_crypto_trace_t *tr = vlib_add_trace (vm, node, b[0],
729  sizeof (*tr));
730  tr->sa_index = sa_index0;
731  }
732 
733  n_left -= 1;
734  next += 1;
735  b += 1;
736  }
737 
740 
741  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
742 
743  return frame->n_vectors;
744 }
745 
747  vlib_node_runtime_t * node,
748  vlib_frame_t * from_frame)
749 {
750  return esp_no_crypto_inline (vm, node, from_frame);
751 }
752 
753 /* *INDENT-OFF* */
755 {
756  .name = "esp4-no-crypto",
757  .vector_size = sizeof (u32),
758  .format_trace = format_esp_no_crypto_trace,
760  .error_strings = esp_no_crypto_error_strings,
761  .n_next_nodes = ESP_NO_CRYPTO_N_NEXT,
762  .next_nodes = {
763  [ESP_NO_CRYPTO_NEXT_DROP] = "ip4-drop",
764  },
765 };
766 
767 VNET_FEATURE_INIT (esp4_no_crypto_tun_feat_node, static) =
768 {
769  .arc_name = "ip4-output",
770  .node_name = "esp4-no-crypto",
771  .runs_before = VNET_FEATURES ("adj-midchain-tx"),
772 };
773 
775  vlib_node_runtime_t * node,
776  vlib_frame_t * from_frame)
777 {
778  return esp_no_crypto_inline (vm, node, from_frame);
779 }
780 
781 /* *INDENT-OFF* */
783 {
784  .name = "esp6-no-crypto",
785  .vector_size = sizeof (u32),
786  .format_trace = format_esp_no_crypto_trace,
788  .error_strings = esp_no_crypto_error_strings,
789  .n_next_nodes = ESP_NO_CRYPTO_N_NEXT,
790  .next_nodes = {
791  [ESP_NO_CRYPTO_NEXT_DROP] = "ip6-drop",
792  },
793 };
794 
795 VNET_FEATURE_INIT (esp6_no_crypto_tun_feat_node, static) =
796 {
797  .arc_name = "ip6-output",
798  .node_name = "esp6-no-crypto",
799  .runs_before = VNET_FEATURES ("adj-midchain-tx"),
800 };
801 /* *INDENT-ON* */
802 
803 /*
804  * fd.io coding-style-patch-verification: ON
805  *
806  * Local Variables:
807  * eval: (c-set-style "gnu")
808  * End:
809  */
u32 vnet_crypto_process_ops(vlib_main_t *vm, vnet_crypto_op_t ops[], u32 n_ops)
Definition: crypto.c:46
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:110
static_always_inline u8 esp_get_ip6_hdr_len(ip6_header_t *ip6)
Definition: esp_encrypt.c:167
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:870
u32 flags
Definition: vhost_user.h:141
#define CLIB_UNUSED(x)
Definition: clib.h:82
ipsec_per_thread_data_t * ptd
Definition: ipsec.h:169
vnet_crypto_op_t * integ_ops
Definition: ipsec.h:89
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
static char * esp_no_crypto_error_strings[]
Definition: esp_encrypt.c:700
static_always_inline void esp_process_ops(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts)
Definition: esp_encrypt.c:205
ipsec_integ_alg_t
Definition: ipsec_sa.h:58
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:572
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:150
u32 thread_index
Definition: main.h:197
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
struct esp_aead_t_ esp_aead_t
AES GCM Additional Authentication data.
vnet_crypto_op_t * crypto_ops
Definition: ipsec.h:88
static void * ip6_ext_next_header(ip6_ext_header_t *ext_hdr)
Definition: ip6_packet.h:550
static u8 * format_esp_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:76
ipsec_integ_alg_t integ_alg
Definition: esp_encrypt.c:71
uword ip_csum_t
Definition: ip_packet.h:219
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
ipsec_crypto_alg_t crypto_alg
Definition: esp_encrypt.c:70
#define VLIB_NODE_FN(node)
Definition: node.h:201
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:468
vnet_crypto_op_id_t integ_op_id
Definition: ipsec_sa.h:129
unsigned char u8
Definition: types.h:56
vlib_node_registration_t esp4_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_tun_node)
Definition: esp_encrypt.c:611
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u32 seq_hi
Definition: ipsec_sa.h:119
static int esp_seq_advance(ipsec_sa_t *sa)
Definition: esp.h:84
vnet_crypto_key_index_t crypto_key_index
Definition: ipsec_sa.h:125
static uword esp_no_crypto_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: esp_encrypt.c:705
#define static_always_inline
Definition: clib.h:99
static_always_inline int esp_trailer_icv_overflow(vlib_node_runtime_t *node, vlib_buffer_t *b, u16 *next, u16 buffer_data_size)
Definition: esp_encrypt.c:192
#define foreach_esp_encrypt_next
Definition: esp_encrypt.c:28
ipsec_main_t ipsec_main
Definition: ipsec.c:28
#define always_inline
Definition: clib.h:98
static_always_inline void esp_fill_udp_hdr(ipsec_sa_t *sa, udp_header_t *udp, u16 len)
Definition: esp_encrypt.c:142
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
static_always_inline void vnet_crypto_op_init(vnet_crypto_op_t *op, vnet_crypto_op_id_t type)
Definition: crypto.h:221
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
static_always_inline u8 ext_hdr_is_pre_esp(u8 nexthdr)
Definition: esp_encrypt.c:149
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:78
unsigned int u32
Definition: types.h:88
static const u8 pad_data[]
Definition: ipsec.h:175
#define VLIB_FRAME_SIZE
Definition: node.h:376
STATIC_ASSERT_SIZEOF(esp_gcm_nonce_t, 12)
static u8 * format_esp_no_crypto_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:678
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
esp_encrypt_error_t
Definition: esp_encrypt.c:49
vlib_node_registration_t esp6_encrypt_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_node)
Definition: esp_encrypt.c:585
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
The identity of a DPO is a combination of its type and its instance number/index of objects of that t...
Definition: dpo.h:170
static_always_inline void * vnet_feature_next_with_data(u32 *next0, vlib_buffer_t *b0, u32 n_data_bytes)
Definition: feature.h:282
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
vlib_node_registration_t esp6_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_tun_node)
Definition: esp_encrypt.c:649
static u8 iv[]
Definition: aes_cbc.c:24
static void esp_aad_fill(vnet_crypto_op_t *op, const esp_header_t *esp, const ipsec_sa_t *sa)
Definition: esp.h:138
uword user_data
Definition: crypto.h:143
vl_api_ip_proto_t protocol
Definition: punt.api:39
u32 salt
Definition: ipsec_sa.h:163
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
vlib_node_registration_t esp4_no_crypto_tun_node
(constructor) VLIB_REGISTER_NODE (esp4_no_crypto_tun_node)
Definition: esp_encrypt.c:754
#define PREDICT_FALSE(x)
Definition: clib.h:111
u32 node_index
Node index.
Definition: node.h:494
static char * esp_encrypt_error_strings[]
Definition: esp_encrypt.c:57
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1150
u8 len
Definition: ip_types.api:90
#define ip6_ext_header_len(p)
Definition: ip6_packet.h:546
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
u16 n_vectors
Definition: node.h:395
#define VNET_CRYPTO_OP_FLAG_INIT_IV
Definition: crypto.h:131
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:312
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
vlib_node_registration_t esp6_no_crypto_tun_node
(constructor) VLIB_REGISTER_NODE (esp6_no_crypto_tun_node)
Definition: esp_encrypt.c:782
vlib_node_registration_t esp4_encrypt_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_node)
Definition: esp_encrypt.c:559
udp_header_t udp_hdr
Definition: ipsec_sa.h:139
#define ARRAY_LEN(x)
Definition: clib.h:62
esp_encrypt_next_t
Definition: esp_encrypt.c:35
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
#define ESP_MAX_BLOCK_SIZE
Definition: esp.h:76
vlib_combined_counter_main_t ipsec_sa_counters
SA packet & bytes counters.
Definition: ipsec_sa.c:27
#define ASSERT(truth)
u32 spi
Definition: ipsec.api:272
static_always_inline u8 * esp_add_footer_and_icv(vlib_buffer_t *b, u8 block_size, u8 icv_sz)
Definition: esp_encrypt.c:94
ip6_header_t ip6_hdr
Definition: ipsec_sa.h:137
ipsec_sa_t * sad
Definition: ipsec.h:97
static_always_inline void esp_update_ip4_hdr(ip4_header_t *ip4, u16 len, int is_transport, int is_udp)
Definition: esp_encrypt.c:116
u32 seq
Definition: esp.h:29
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
u32 spi
Definition: esp.h:26
vnet_crypto_key_index_t integ_key_index
Definition: ipsec_sa.h:126
#define VNET_FEATURES(...)
Definition: feature.h:435
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static_always_inline void clib_memcpy_le64(u8 *dst, u8 *src, u8 len)
Definition: string.h:283
Definition: defs.h:47
u16 payload_length
Definition: ip6_packet.h:374
index_t dpoi_index
the index of objects of that type
Definition: dpo.h:186
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
ipsec_crypto_alg_t
Definition: ipsec_sa.h:36
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:269
static uword esp_encrypt_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ip6, int is_tun)
Definition: esp_encrypt.c:240
u64 gcm_iv_counter
Definition: ipsec_sa.h:164
vnet_crypto_op_id_t crypto_enc_op_id
Definition: ipsec_sa.h:127
vnet_crypto_op_status_t status
Definition: crypto.h:129
#define vnet_buffer(b)
Definition: buffer.h:361
dpo_id_t dpo
Definition: ipsec_sa.h:123
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:146
ip4_header_t ip4_hdr
Definition: ipsec_sa.h:136
u8 crypto_block_size
Definition: ipsec_sa.h:115
u16 dpoi_next_node
The next VLIB node to follow.
Definition: dpo.h:182
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:235
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
VNET_FEATURE_INIT(esp4_encrypt_tun_feat_node, static)
u8 crypto_iv_size
Definition: ipsec_sa.h:114
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 salt
Definition: ipsec.api:287
#define foreach_esp_encrypt_error
Definition: esp_encrypt.c:42
u8 integ_icv_size
Definition: ipsec_sa.h:116
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:275
static_always_inline void clib_memcpy_le32(u8 *dst, u8 *src, u8 len)
Definition: string.h:289