adj_nbr.c

Go to the documentation of this file.

/*
 * Copyright (c) 2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vnet/adj/adj_nbr.h>
#include <vnet/adj/adj_internal.h>
#include <vnet/ethernet/arp_packet.h>
#include <vnet/fib/fib_walk.h>

/*
 * Vector Hash tables of neighbour (traditional) adjacencies
 *  Key: interface(for the vector index), address (and its proto),
 *       link-type/ether-type.
 */
static BVT(clib_bihash) **adj_nbr_tables[FIB_PROTOCOL_MAX];

// FIXME SIZE APPROPRIATELY. ASK DAVEB.
#define ADJ_NBR_DEFAULT_HASH_NUM_BUCKETS (64 * 64)
#define ADJ_NBR_DEFAULT_HASH_MEMORY_SIZE (32<<20)


#define ADJ_NBR_SET_KEY(_key, _lt, _nh)         \
{                                               \
    _key.key[0] = (_nh)->as_u64[0];             \
    _key.key[1] = (_nh)->as_u64[1];             \
    _key.key[2] = (_lt);                        \
}

#define ADJ_NBR_ITF_OK(_proto, _itf)                    \
    (((_itf) < vec_len(adj_nbr_tables[_proto])) &&      \
     (NULL != adj_nbr_tables[_proto][sw_if_index]))

static void
adj_nbr_insert (fib_protocol_t nh_proto,
                vnet_link_t link_type,
                const ip46_address_t *nh_addr,
                u32 sw_if_index,
                adj_index_t adj_index)
{
    BVT(clib_bihash_kv) kv;

    if (sw_if_index >= vec_len(adj_nbr_tables[nh_proto]))
    {
        vec_validate(adj_nbr_tables[nh_proto], sw_if_index);
    }
    if (NULL == adj_nbr_tables[nh_proto][sw_if_index])
    {
        adj_nbr_tables[nh_proto][sw_if_index] =
            clib_mem_alloc_aligned(sizeof(BVT(clib_bihash)),
                                   CLIB_CACHE_LINE_BYTES);
        memset(adj_nbr_tables[nh_proto][sw_if_index],
               0,
               sizeof(BVT(clib_bihash)));

        BV(clib_bihash_init) (adj_nbr_tables[nh_proto][sw_if_index],
                              "Adjacency Neighbour table",
                              ADJ_NBR_DEFAULT_HASH_NUM_BUCKETS,
                              ADJ_NBR_DEFAULT_HASH_MEMORY_SIZE);
    }

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);
    kv.value = adj_index;

    BV(clib_bihash_add_del) (adj_nbr_tables[nh_proto][sw_if_index], &kv, 1);
}

void
adj_nbr_remove (fib_protocol_t nh_proto,
                vnet_link_t link_type,
                const ip46_address_t *nh_addr,
                u32 sw_if_index)
{
    BVT(clib_bihash_kv) kv;

    if (!ADJ_NBR_ITF_OK(nh_proto, sw_if_index))
        return;

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);

    BV(clib_bihash_add_del) (adj_nbr_tables[nh_proto][sw_if_index], &kv, 0);
}

static adj_index_t
adj_nbr_find (fib_protocol_t nh_proto,
              vnet_link_t link_type,
              const ip46_address_t *nh_addr,
              u32 sw_if_index)
{
    BVT(clib_bihash_kv) kv;

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);

    if (!ADJ_NBR_ITF_OK(nh_proto, sw_if_index))
        return (ADJ_INDEX_INVALID);

    if (BV(clib_bihash_search)(adj_nbr_tables[nh_proto][sw_if_index],
                               &kv, &kv) < 0)
    {
        return (ADJ_INDEX_INVALID);
    }
    else
    {
        return (kv.value);
    }
}

static inline u32
adj_get_nd_node (fib_protocol_t proto)
{
    switch (proto) {
    case FIB_PROTOCOL_IP4:
        return (ip4_arp_node.index);
    case FIB_PROTOCOL_IP6:
        return (ip6_discover_neighbor_node.index);
    case FIB_PROTOCOL_MPLS:
        break;
    }
    ASSERT(0);
    return (ip4_arp_node.index);
}

static ip_adjacency_t*
adj_nbr_alloc (fib_protocol_t nh_proto,
               vnet_link_t link_type,
               const ip46_address_t *nh_addr,
               u32 sw_if_index)
{
    ip_adjacency_t *adj;

    adj = adj_alloc(nh_proto);

    adj_nbr_insert(nh_proto, link_type, nh_addr,
                   sw_if_index,
                   adj_get_index(adj));

    /*
     * since we just added the ADJ we have no rewrite string for it,
     * so its for ARP
     */
    adj->lookup_next_index = IP_LOOKUP_NEXT_ARP;
    adj->sub_type.nbr.next_hop = *nh_addr;
    adj->ia_link = link_type;
    adj->ia_nh_proto = nh_proto;
    adj->rewrite_header.sw_if_index = sw_if_index;
    memset(&adj->sub_type.midchain.next_dpo, 0,
           sizeof(adj->sub_type.midchain.next_dpo));

    return (adj);
}

/*
 * adj_add_for_nbr
 *
 * Add an adjacency for the neighbour requested.
 *
 * The key for an adj is:
 *   - the Next-hops protocol (i.e. v4 or v6)
 *   - the address of the next-hop
 *   - the interface the next-hop is reachable through
 */
adj_index_t
adj_nbr_add_or_lock (fib_protocol_t nh_proto,
                     vnet_link_t link_type,
                     const ip46_address_t *nh_addr,
                     u32 sw_if_index)
{
    adj_index_t adj_index;
    ip_adjacency_t *adj;

    adj_index = adj_nbr_find(nh_proto, link_type, nh_addr, sw_if_index);

    if (ADJ_INDEX_INVALID == adj_index)
    {
        vnet_main_t *vnm;

        vnm = vnet_get_main();
        adj = adj_nbr_alloc(nh_proto, link_type, nh_addr, sw_if_index);
        adj_index = adj_get_index(adj);
        adj_lock(adj_index);

        vnet_rewrite_init(vnm, sw_if_index,
                          adj_get_nd_node(nh_proto),
                          vnet_tx_node_index_for_sw_interface(vnm, sw_if_index),
                          &adj->rewrite_header);

        /*
         * we need a rewrite where the destination IP address is converted
         * to the appropriate link-layer address. This is interface specific.
         * So ask the interface to do it.
         */
        vnet_update_adjacency_for_sw_interface(vnm, sw_if_index, adj_index);
    }
    else
    {
        adj_lock(adj_index);
    }

    return (adj_index);
}

adj_index_t
adj_nbr_add_or_lock_w_rewrite (fib_protocol_t nh_proto,
                               vnet_link_t link_type,
                               const ip46_address_t *nh_addr,
                               u32 sw_if_index,
                               u8 *rewrite)
{
    adj_index_t adj_index;
    ip_adjacency_t *adj;

    adj_index = adj_nbr_find(nh_proto, link_type, nh_addr, sw_if_index);

    if (ADJ_INDEX_INVALID == adj_index)
    {
        adj = adj_nbr_alloc(nh_proto, link_type, nh_addr, sw_if_index);
        adj->rewrite_header.sw_if_index = sw_if_index;
    }
    else
    {
        adj = adj_get(adj_index);
    }

    adj_lock(adj_get_index(adj));
    adj_nbr_update_rewrite(adj_get_index(adj),
                           ADJ_NBR_REWRITE_FLAG_COMPLETE,
                           rewrite);

    return (adj_get_index(adj));
}

/**
 * adj_nbr_update_rewrite
 *
 * Update the adjacency's rewrite string. A NULL string implies the
 * rewirte is reset (i.e. when ARP/ND etnry is gone).
 * NB: the adj being updated may be handling traffic in the DP.
 */
void
adj_nbr_update_rewrite (adj_index_t adj_index,
                        adj_nbr_rewrite_flag_t flags,
                        u8 *rewrite)
{
    ip_adjacency_t *adj;
    u32 old_next;

    ASSERT(ADJ_INDEX_INVALID != adj_index);

    adj = adj_get(adj_index);
    old_next = adj->lookup_next_index;

    if (flags & ADJ_NBR_REWRITE_FLAG_COMPLETE)
    {
        /*
         * update the adj's rewrite string and build the arc
         * from the rewrite node to the interface's TX node
         */
        adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_REWRITE,
                                        adj_get_rewrite_node(adj->ia_link),
                                        vnet_tx_node_index_for_sw_interface(
                                            vnet_get_main(),
                                            adj->rewrite_header.sw_if_index),
                                        rewrite);
    }
    else
    {
        adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_ARP,
                                        adj_get_nd_node(adj->ia_nh_proto),
                                        vnet_tx_node_index_for_sw_interface(
                                            vnet_get_main(),
                                            adj->rewrite_header.sw_if_index),
                                        rewrite);
    }

    if (old_next != adj->lookup_next_index)
    {
        /*
         * time for walkies fido.
         * The link type MPLS Adj never has children. So if it is this adj
         * that is updated, we need to walk from its IP sibling.
         */
        if (VNET_LINK_MPLS == adj->ia_link)
        {
            adj_index = adj_nbr_find(adj->ia_nh_proto,
                                     fib_proto_to_link(adj->ia_nh_proto),
                                     &adj->sub_type.nbr.next_hop,
                                     adj->rewrite_header.sw_if_index);

            ASSERT(ADJ_INDEX_INVALID != adj_index);
        }

        fib_node_back_walk_ctx_t bw_ctx = {
            .fnbw_reason = FIB_NODE_BW_REASON_FLAG_ADJ_UPDATE,
            /*
             * This walk only needs to go back one level, but there is no control
             * here. the first receiving fib_entry_t will quash the walk
             */
        };

        fib_walk_sync(FIB_NODE_TYPE_ADJ, adj_index, &bw_ctx);
    }
}

/**
 * adj_nbr_update_rewrite_internal
 *
 * Update the adjacency's rewrite string. A NULL string implies the
 * rewirte is reset (i.e. when ARP/ND etnry is gone).
 * NB: the adj being updated may be handling traffic in the DP.
 */
void
adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
                                 u32 adj_next_index,
                                 u32 this_node,
                                 u32 next_node,
                                 u8 *rewrite)
{
    vlib_main_t * vm = vlib_get_main();

    /*
     * Updating a rewrite string is not atomic;
     *  - the rewrite string is too long to write in one instruction
     *  - when swapping from incomplete to complete, we also need to update
     *    the VLIB graph next-index.
     * ideally we would only want to suspend forwarding via this adj whilst we
     * do this, but we do not have that level of granularity - it's suspend all
     * worker threads or nothing.
     * The other chioces are:
     *  - to mark the adj down and back walk so child load-balances drop this adj
     *    from the set.
     *  - update the next_node index of this adj to point to error-drop
     * both of which will mean for MAC change we will drop for this adj
     * which is not acceptable.
     * So the pause all threads is preferable. We don't update MAC addresses often
     * so it's no big deal.
     */
    vlib_worker_thread_barrier_sync(vm);

    adj->lookup_next_index = adj_next_index;

    if (NULL != rewrite)
    {
        /*
         * new rewrite provided.
         * fill in the adj's rewrite string, and build the VLIB graph arc.
         */
        vnet_rewrite_set_data_internal(&adj->rewrite_header,
                                       sizeof(adj->rewrite_data),
                                       rewrite,
                                       vec_len(rewrite));

        adj->rewrite_header.node_index = this_node;
        adj->rewrite_header.next_index = vlib_node_add_next (vlib_get_main(),
                                                             this_node,
                                                             next_node);

        vec_free(rewrite);
    }
    else
    {
        vnet_rewrite_clear_data_internal(&adj->rewrite_header,
                                         sizeof(adj->rewrite_data));
    }

    /*
     * done with the rewirte update - let the workers loose.
     */
    vlib_worker_thread_barrier_release(vm);
}

typedef struct adj_db_count_ctx_t_ {
    u64 count;
} adj_db_count_ctx_t;

static void
adj_db_count (BVT(clib_bihash_kv) * kvp,
              void *arg)
{
    adj_db_count_ctx_t * ctx = arg;
    ctx->count++;
}

u32
adj_nbr_db_size (void)
{
    adj_db_count_ctx_t ctx = {
        .count = 0,
    };
    fib_protocol_t proto;
    u32 sw_if_index = 0;

    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
        vec_foreach_index(sw_if_index, adj_nbr_tables[proto])
        {
            if (NULL != adj_nbr_tables[proto][sw_if_index])
            {
                BV(clib_bihash_foreach_key_value_pair) (
                    adj_nbr_tables[proto][sw_if_index],
                    adj_db_count,
                    &ctx);
            }
        }
    }
    return (ctx.count);
}

/**
 * @brief Context for a walk of the adjacency neighbour DB
 */
typedef struct adj_walk_ctx_t_
{
    adj_walk_cb_t awc_cb;
    void *awc_ctx;
} adj_walk_ctx_t;

static void
adj_nbr_walk_cb (BVT(clib_bihash_kv) * kvp,
                 void *arg)
{
    adj_walk_ctx_t *ctx = arg;

    // FIXME: can't stop early...
    ctx->awc_cb(kvp->value, ctx->awc_ctx);
}

void
adj_nbr_walk (u32 sw_if_index,
              fib_protocol_t adj_nh_proto,
              adj_walk_cb_t cb,
              void *ctx)
{
    if (!ADJ_NBR_ITF_OK(adj_nh_proto, sw_if_index))
        return;

    adj_walk_ctx_t awc = {
        .awc_ctx = ctx,
        .awc_cb = cb,
    };

    BV(clib_bihash_foreach_key_value_pair) (
        adj_nbr_tables[adj_nh_proto][sw_if_index],
        adj_nbr_walk_cb,
        &awc);
}

/**
 * @brief Context for a walk of the adjacency neighbour DB
 */
typedef struct adj_walk_nh_ctx_t_
{
    adj_walk_cb_t awc_cb;
    void *awc_ctx;
    const ip46_address_t *awc_nh;
} adj_walk_nh_ctx_t;

static void
adj_nbr_walk_nh_cb (BVT(clib_bihash_kv) * kvp,
                    void *arg)
{
    ip_adjacency_t *adj;
    adj_walk_nh_ctx_t *ctx = arg;

    adj = adj_get(kvp->value);

    if (!ip46_address_cmp(&adj->sub_type.nbr.next_hop, ctx->awc_nh)) 
        ctx->awc_cb(kvp->value, ctx->awc_ctx);
}

/**
 * @brief Walk adjacencies on a link with a given v4 next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh4 (u32 sw_if_index,
                 const ip4_address_t *addr,
                 adj_walk_cb_t cb,
                 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(FIB_PROTOCOL_IP4, sw_if_index))
        return;

    ip46_address_t nh = {
        .ip4 = *addr,
    };

    adj_walk_nh_ctx_t awc = {
        .awc_ctx = ctx,
        .awc_cb = cb,
        .awc_nh = &nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
        adj_nbr_tables[FIB_PROTOCOL_IP4][sw_if_index],
        adj_nbr_walk_nh_cb,
        &awc);
}

/**
 * @brief Walk adjacencies on a link with a given v6 next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh6 (u32 sw_if_index,
                 const ip6_address_t *addr,
                 adj_walk_cb_t cb,
                 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(FIB_PROTOCOL_IP6, sw_if_index))
        return;

    ip46_address_t nh = {
        .ip6 = *addr,
    };

    adj_walk_nh_ctx_t awc = {
        .awc_ctx = ctx,
        .awc_cb = cb,
        .awc_nh = &nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
        adj_nbr_tables[FIB_PROTOCOL_IP6][sw_if_index],
        adj_nbr_walk_nh_cb,
        &awc);
}

/**
 * @brief Walk adjacencies on a link with a given next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh (u32 sw_if_index,
                 fib_protocol_t adj_nh_proto,
                 const ip46_address_t *nh,
                 adj_walk_cb_t cb,
                 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(adj_nh_proto, sw_if_index))
        return;

    adj_walk_nh_ctx_t awc = {
        .awc_ctx = ctx,
        .awc_cb = cb,
        .awc_nh = nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
        adj_nbr_tables[adj_nh_proto][sw_if_index],
        adj_nbr_walk_nh_cb,
        &awc);
}

/**
 * Context for the state change walk of the DB
 */
typedef struct adj_nbr_interface_state_change_ctx_t_
{
    /**
     * Flags passed from the vnet notifiy function
     */
    int flags;
} adj_nbr_interface_state_change_ctx_t;

static adj_walk_rc_t
adj_nbr_interface_state_change_one (adj_index_t ai,
                                    void *arg)
{
    /*
     * Back walk the graph to inform the forwarding entries
     * that this interface state has changed.
     */
    adj_nbr_interface_state_change_ctx_t *ctx = arg;

    fib_node_back_walk_ctx_t bw_ctx = {
        .fnbw_reason = (ctx->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ?
                        FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
                        FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
    };

    fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);

    return (ADJ_WALK_RC_CONTINUE);
}

static clib_error_t *
adj_nbr_interface_state_change (vnet_main_t * vnm,
                                u32 sw_if_index,
                                u32 flags)
{
    fib_protocol_t proto;

    /*
     * walk each adj on the interface and trigger a walk from that adj
     */
    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
        adj_nbr_interface_state_change_ctx_t ctx = {
            .flags = flags,
        };

        adj_nbr_walk(sw_if_index, proto,
                     adj_nbr_interface_state_change_one,
                     &ctx);
    }

    return (NULL);
}

VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(adj_nbr_interface_state_change);

static adj_walk_rc_t
adj_nbr_interface_delete_one (adj_index_t ai,
                              void *arg)
{
    /*
     * Back walk the graph to inform the forwarding entries
     * that this interface has been deleted.
     */
    fib_node_back_walk_ctx_t bw_ctx = {
        .fnbw_reason = FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE,
    };

    fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);

    return (ADJ_WALK_RC_CONTINUE);
}

/**
 * adj_nbr_interface_add_del
 *
 * Registered to receive interface Add and delete notifications
 */
static clib_error_t *
adj_nbr_interface_add_del (vnet_main_t * vnm,
                           u32 sw_if_index,
                           u32 is_add)
{
    fib_protocol_t proto;

    if (is_add)
    {
        /*
         * not interested in interface additions. we will not back walk
         * to resolve paths through newly added interfaces. Why? The control
         * plane should have the brains to add interfaces first, then routes.
         * So the case where there are paths with a interface that matches
         * one just created is the case where the path resolved through an
         * interface that was deleted, and still has not been removed. The
         * new interface added, is NO GUARANTEE that the interface being
         * added now, even though it may have the same sw_if_index, is the
         * same interface that the path needs. So tough!
         * If the control plane wants these routes to resolve it needs to
         * remove and add them again.
         */
        return (NULL);
    }

    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
        adj_nbr_walk(sw_if_index, proto,
                     adj_nbr_interface_delete_one,
                     NULL);
    }

    return (NULL);
   
}

VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_nbr_interface_add_del);


static adj_walk_rc_t
adj_nbr_show_one (adj_index_t ai,
                  void *arg)
{
    vlib_cli_output (arg, "[@%d]  %U",
                     ai,
                     format_ip_adjacency, ai,
                     FORMAT_IP_ADJACENCY_NONE);

    return (ADJ_WALK_RC_CONTINUE);
}

static clib_error_t *
adj_nbr_show (vlib_main_t * vm,
              unformat_input_t * input,
              vlib_cli_command_t * cmd)
{
    adj_index_t ai = ADJ_INDEX_INVALID;
    u32 sw_if_index = ~0;

    while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
    {
        if (unformat (input, "%d", &ai))
            ;
        else if (unformat (input, "%U",
                           unformat_vnet_sw_interface, vnet_get_main(),
                           &sw_if_index))
            ;
        else
            break;
    }

    if (ADJ_INDEX_INVALID != ai)
    {
        vlib_cli_output (vm, "[@%d] %U",
                         ai,
                         format_ip_adjacency, ai,
                         FORMAT_IP_ADJACENCY_DETAIL);
    }
    else if (~0 != sw_if_index)
    {
        fib_protocol_t proto;

        for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
        {
            adj_nbr_walk(sw_if_index, proto,
                         adj_nbr_show_one,
                         vm);
        }
    }
    else
    {
        fib_protocol_t proto;

        for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
        {
            vec_foreach_index(sw_if_index, adj_nbr_tables[proto])
            {
                adj_nbr_walk(sw_if_index, proto,
                             adj_nbr_show_one,
                             vm);
            }
        }
    }

    return 0;
}

/*?
 * Show all neighbour adjacencies.
 * @cliexpar
 * @cliexstart{sh adj nbr}
 * [@2] ipv4 via 1.0.0.2 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@3] mpls via 1.0.0.2 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@4] ipv4 via 1.0.0.3 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@5] mpls via 1.0.0.3 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * @cliexend
 ?*/
VLIB_CLI_COMMAND (ip4_show_fib_command, static) = {
    .path = "show adj nbr",
    .short_help = "show adj nbr [<adj_index>] [interface]",
    .function = adj_nbr_show,
};

static ip46_type_t
adj_proto_to_46 (fib_protocol_t proto)
{
    switch (proto)
    {
    case FIB_PROTOCOL_IP4:
        return (IP46_TYPE_IP4);
    case FIB_PROTOCOL_IP6:
        return (IP46_TYPE_IP6);
    default:
        return (IP46_TYPE_IP4);
    }
    return (IP46_TYPE_IP4);
}

u8*
format_adj_nbr_incomplete (u8* s, va_list *ap)
{
    index_t index = va_arg(*ap, index_t);
    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
    vnet_main_t * vnm = vnet_get_main();
    ip_adjacency_t * adj = adj_get(index);

    s = format (s, "arp-%U", format_vnet_link, adj->ia_link);
    s = format (s, ": via %U",
                format_ip46_address, &adj->sub_type.nbr.next_hop,
                adj_proto_to_46(adj->ia_nh_proto));
    s = format (s, " %U",
                format_vnet_sw_interface_name,
                vnm,
                vnet_get_sw_interface(vnm,
                                      adj->rewrite_header.sw_if_index));

    return (s);
}

u8*
format_adj_nbr (u8* s, va_list *ap)
{
    index_t index = va_arg(*ap, index_t);
    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
    vnet_main_t * vnm = vnet_get_main();
    ip_adjacency_t * adj = adj_get(index);

    s = format (s, "%U", format_vnet_link, adj->ia_link);
    s = format (s, " via %U ",
                format_ip46_address, &adj->sub_type.nbr.next_hop,
                adj_proto_to_46(adj->ia_nh_proto));
    s = format (s, "%U",
                format_vnet_rewrite,
                vnm->vlib_main, &adj->rewrite_header, sizeof (adj->rewrite_data), 0);

    return (s);
}

static void
adj_dpo_lock (dpo_id_t *dpo)
{
    adj_lock(dpo->dpoi_index);
}
static void
adj_dpo_unlock (dpo_id_t *dpo)
{
    adj_unlock(dpo->dpoi_index);
}

static void
adj_mem_show (void)
{
    fib_show_memory_usage("Adjacency",
                          pool_elts(adj_pool),
                          pool_len(adj_pool),
                          sizeof(ip_adjacency_t));
}

const static dpo_vft_t adj_nbr_dpo_vft = {
    .dv_lock = adj_dpo_lock,
    .dv_unlock = adj_dpo_unlock,
    .dv_format = format_adj_nbr,
    .dv_mem_show = adj_mem_show,
};
const static dpo_vft_t adj_nbr_incompl_dpo_vft = {
    .dv_lock = adj_dpo_lock,
    .dv_unlock = adj_dpo_unlock,
    .dv_format = format_adj_nbr_incomplete,
};

/**
 * @brief The per-protocol VLIB graph nodes that are assigned to an adjacency
 *        object.
 *
 * this means that these graph nodes are ones from which a nbr is the
 * parent object in the DPO-graph.
 */
const static char* const nbr_ip4_nodes[] =
{
    "ip4-rewrite-transit",
    NULL,
};
const static char* const nbr_ip6_nodes[] =
{
    "ip6-rewrite",
    NULL,
};
const static char* const nbr_mpls_nodes[] =
{
    "mpls-output",
    NULL,
};
const static char* const nbr_ethernet_nodes[] =
{
    "adj-l2-rewrite",
    NULL,
};
const static char* const * const nbr_nodes[DPO_PROTO_NUM] =
{
    [DPO_PROTO_IP4]  = nbr_ip4_nodes,
    [DPO_PROTO_IP6]  = nbr_ip6_nodes,
    [DPO_PROTO_MPLS] = nbr_mpls_nodes,
    [DPO_PROTO_ETHERNET] = nbr_ethernet_nodes,
};

const static char* const nbr_incomplete_ip4_nodes[] =
{
    "ip4-arp",
    NULL,
};
const static char* const nbr_incomplete_ip6_nodes[] =
{
    "ip6-discover-neighbor",
    NULL,
};
const static char* const nbr_incomplete_mpls_nodes[] =
{
    "mpls-adj-incomplete",
    NULL,
};

const static char* const * const nbr_incomplete_nodes[DPO_PROTO_NUM] =
{
    [DPO_PROTO_IP4]  = nbr_incomplete_ip4_nodes,
    [DPO_PROTO_IP6]  = nbr_incomplete_ip6_nodes,
    [DPO_PROTO_MPLS] = nbr_incomplete_mpls_nodes,
};

void
adj_nbr_module_init (void)
{
    dpo_register(DPO_ADJACENCY,
                 &adj_nbr_dpo_vft,
                 nbr_nodes);
    dpo_register(DPO_ADJACENCY_INCOMPLETE,
                 &adj_nbr_incompl_dpo_vft,
                 nbr_incomplete_nodes);
}