Commit 16c6cf8bb471392fd09b48b7c27e7d83a446b4bc

Authored by Stephen Hemminger
Committed by David S. Miller
1 parent 977750076d

ipv4: fib table algorithm performance improvement

The FIB algorithim for IPV4 is set at compile time, but kernel goes through
the overhead of function call indirection at runtime. Save some
cycles by turning the indirect calls to direct calls to either
hash or trie code.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

Showing 5 changed files with 48 additions and 56 deletions Side-by-side Diff

include/net/ip_fib.h
... ... @@ -144,18 +144,21 @@
144 144 struct hlist_node tb_hlist;
145 145 u32 tb_id;
146 146 int tb_default;
147   - int (*tb_lookup)(struct fib_table *tb, const struct flowi *flp, struct fib_result *res);
148   - int (*tb_insert)(struct fib_table *, struct fib_config *);
149   - int (*tb_delete)(struct fib_table *, struct fib_config *);
150   - int (*tb_dump)(struct fib_table *table, struct sk_buff *skb,
151   - struct netlink_callback *cb);
152   - int (*tb_flush)(struct fib_table *table);
153   - void (*tb_select_default)(struct fib_table *table,
154   - const struct flowi *flp, struct fib_result *res);
155   -
156 147 unsigned char tb_data[0];
157 148 };
158 149  
  150 +extern int fib_table_lookup(struct fib_table *tb, const struct flowi *flp,
  151 + struct fib_result *res);
  152 +extern int fib_table_insert(struct fib_table *, struct fib_config *);
  153 +extern int fib_table_delete(struct fib_table *, struct fib_config *);
  154 +extern int fib_table_dump(struct fib_table *table, struct sk_buff *skb,
  155 + struct netlink_callback *cb);
  156 +extern int fib_table_flush(struct fib_table *table);
  157 +extern void fib_table_select_default(struct fib_table *table,
  158 + const struct flowi *flp,
  159 + struct fib_result *res);
  160 +
  161 +
159 162 #ifndef CONFIG_IP_MULTIPLE_TABLES
160 163  
161 164 #define TABLE_LOCAL_INDEX 0
162 165  
... ... @@ -182,11 +185,11 @@
182 185 struct fib_table *table;
183 186  
184 187 table = fib_get_table(net, RT_TABLE_LOCAL);
185   - if (!table->tb_lookup(table, flp, res))
  188 + if (!fib_table_lookup(table, flp, res))
186 189 return 0;
187 190  
188 191 table = fib_get_table(net, RT_TABLE_MAIN);
189   - if (!table->tb_lookup(table, flp, res))
  192 + if (!fib_table_lookup(table, flp, res))
190 193 return 0;
191 194 return -ENETUNREACH;
192 195 }
net/ipv4/fib_frontend.c
... ... @@ -125,7 +125,7 @@
125 125 #endif
126 126 tb = fib_get_table(net, table);
127 127 if (FIB_RES_GW(*res) && FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
128   - tb->tb_select_default(tb, flp, res);
  128 + fib_table_select_default(tb, flp, res);
129 129 }
130 130  
131 131 static void fib_flush(struct net *net)
... ... @@ -139,7 +139,7 @@
139 139 for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
140 140 head = &net->ipv4.fib_table_hash[h];
141 141 hlist_for_each_entry(tb, node, head, tb_hlist)
142   - flushed += tb->tb_flush(tb);
  142 + flushed += fib_table_flush(tb);
143 143 }
144 144  
145 145 if (flushed)
... ... @@ -162,7 +162,7 @@
162 162 #endif
163 163  
164 164 local_table = fib_get_table(net, RT_TABLE_LOCAL);
165   - if (!local_table || local_table->tb_lookup(local_table, &fl, &res))
  165 + if (!local_table || fib_table_lookup(local_table, &fl, &res))
166 166 return NULL;
167 167 if (res.type != RTN_LOCAL)
168 168 goto out;
... ... @@ -200,7 +200,7 @@
200 200 local_table = fib_get_table(net, RT_TABLE_LOCAL);
201 201 if (local_table) {
202 202 ret = RTN_UNICAST;
203   - if (!local_table->tb_lookup(local_table, &fl, &res)) {
  203 + if (!fib_table_lookup(local_table, &fl, &res)) {
204 204 if (!dev || dev == res.fi->fib_dev)
205 205 ret = res.type;
206 206 fib_res_put(&res);
207 207  
... ... @@ -473,13 +473,13 @@
473 473 if (cmd == SIOCDELRT) {
474 474 tb = fib_get_table(net, cfg.fc_table);
475 475 if (tb)
476   - err = tb->tb_delete(tb, &cfg);
  476 + err = fib_table_delete(tb, &cfg);
477 477 else
478 478 err = -ESRCH;
479 479 } else {
480 480 tb = fib_new_table(net, cfg.fc_table);
481 481 if (tb)
482   - err = tb->tb_insert(tb, &cfg);
  482 + err = fib_table_insert(tb, &cfg);
483 483 else
484 484 err = -ENOBUFS;
485 485 }
... ... @@ -594,7 +594,7 @@
594 594 goto errout;
595 595 }
596 596  
597   - err = tb->tb_delete(tb, &cfg);
  597 + err = fib_table_delete(tb, &cfg);
598 598 errout:
599 599 return err;
600 600 }
... ... @@ -616,7 +616,7 @@
616 616 goto errout;
617 617 }
618 618  
619   - err = tb->tb_insert(tb, &cfg);
  619 + err = fib_table_insert(tb, &cfg);
620 620 errout:
621 621 return err;
622 622 }
... ... @@ -647,7 +647,7 @@
647 647 if (dumped)
648 648 memset(&cb->args[2], 0, sizeof(cb->args) -
649 649 2 * sizeof(cb->args[0]));
650   - if (tb->tb_dump(tb, skb, cb) < 0)
  650 + if (fib_table_dump(tb, skb, cb) < 0)
651 651 goto out;
652 652 dumped = 1;
653 653 next:
654 654  
... ... @@ -701,9 +701,9 @@
701 701 cfg.fc_scope = RT_SCOPE_HOST;
702 702  
703 703 if (cmd == RTM_NEWROUTE)
704   - tb->tb_insert(tb, &cfg);
  704 + fib_table_insert(tb, &cfg);
705 705 else
706   - tb->tb_delete(tb, &cfg);
  706 + fib_table_delete(tb, &cfg);
707 707 }
708 708  
709 709 void fib_add_ifaddr(struct in_ifaddr *ifa)
... ... @@ -832,7 +832,7 @@
832 832 local_bh_disable();
833 833  
834 834 frn->tb_id = tb->tb_id;
835   - frn->err = tb->tb_lookup(tb, &fl, &res);
  835 + frn->err = fib_table_lookup(tb, &fl, &res);
836 836  
837 837 if (!frn->err) {
838 838 frn->prefixlen = res.prefixlen;
... ... @@ -1009,7 +1009,7 @@
1009 1009 head = &net->ipv4.fib_table_hash[i];
1010 1010 hlist_for_each_entry_safe(tb, node, tmp, head, tb_hlist) {
1011 1011 hlist_del(node);
1012   - tb->tb_flush(tb);
  1012 + fib_table_flush(tb);
1013 1013 kfree(tb);
1014 1014 }
1015 1015 }
... ... @@ -242,8 +242,8 @@
242 242 return fz;
243 243 }
244 244  
245   -static int
246   -fn_hash_lookup(struct fib_table *tb, const struct flowi *flp, struct fib_result *res)
  245 +int fib_table_lookup(struct fib_table *tb,
  246 + const struct flowi *flp, struct fib_result *res)
247 247 {
248 248 int err;
249 249 struct fn_zone *fz;
... ... @@ -274,8 +274,8 @@
274 274 return err;
275 275 }
276 276  
277   -static void
278   -fn_hash_select_default(struct fib_table *tb, const struct flowi *flp, struct fib_result *res)
  277 +void fib_table_select_default(struct fib_table *tb,
  278 + const struct flowi *flp, struct fib_result *res)
279 279 {
280 280 int order, last_idx;
281 281 struct hlist_node *node;
... ... @@ -366,7 +366,7 @@
366 366 return NULL;
367 367 }
368 368  
369   -static int fn_hash_insert(struct fib_table *tb, struct fib_config *cfg)
  369 +int fib_table_insert(struct fib_table *tb, struct fib_config *cfg)
370 370 {
371 371 struct fn_hash *table = (struct fn_hash *) tb->tb_data;
372 372 struct fib_node *new_f = NULL;
... ... @@ -544,8 +544,7 @@
544 544 return err;
545 545 }
546 546  
547   -
548   -static int fn_hash_delete(struct fib_table *tb, struct fib_config *cfg)
  547 +int fib_table_delete(struct fib_table *tb, struct fib_config *cfg)
549 548 {
550 549 struct fn_hash *table = (struct fn_hash *)tb->tb_data;
551 550 struct fib_node *f;
... ... @@ -662,7 +661,7 @@
662 661 return found;
663 662 }
664 663  
665   -static int fn_hash_flush(struct fib_table *tb)
  664 +int fib_table_flush(struct fib_table *tb)
666 665 {
667 666 struct fn_hash *table = (struct fn_hash *) tb->tb_data;
668 667 struct fn_zone *fz;
... ... @@ -743,7 +742,8 @@
743 742 return skb->len;
744 743 }
745 744  
746   -static int fn_hash_dump(struct fib_table *tb, struct sk_buff *skb, struct netlink_callback *cb)
  745 +int fib_table_dump(struct fib_table *tb, struct sk_buff *skb,
  746 + struct netlink_callback *cb)
747 747 {
748 748 int m, s_m;
749 749 struct fn_zone *fz;
... ... @@ -787,12 +787,7 @@
787 787  
788 788 tb->tb_id = id;
789 789 tb->tb_default = -1;
790   - tb->tb_lookup = fn_hash_lookup;
791   - tb->tb_insert = fn_hash_insert;
792   - tb->tb_delete = fn_hash_delete;
793   - tb->tb_flush = fn_hash_flush;
794   - tb->tb_select_default = fn_hash_select_default;
795   - tb->tb_dump = fn_hash_dump;
  790 +
796 791 memset(tb->tb_data, 0, sizeof(struct fn_hash));
797 792 return tb;
798 793 }
net/ipv4/fib_rules.c
... ... @@ -94,7 +94,7 @@
94 94 if ((tbl = fib_get_table(rule->fr_net, rule->table)) == NULL)
95 95 goto errout;
96 96  
97   - err = tbl->tb_lookup(tbl, flp, (struct fib_result *) arg->result);
  97 + err = fib_table_lookup(tbl, flp, (struct fib_result *) arg->result);
98 98 if (err > 0)
99 99 err = -EAGAIN;
100 100 errout:
... ... @@ -1174,7 +1174,7 @@
1174 1174 /*
1175 1175 * Caller must hold RTNL.
1176 1176 */
1177   -static int fn_trie_insert(struct fib_table *tb, struct fib_config *cfg)
  1177 +int fib_table_insert(struct fib_table *tb, struct fib_config *cfg)
1178 1178 {
1179 1179 struct trie *t = (struct trie *) tb->tb_data;
1180 1180 struct fib_alias *fa, *new_fa;
... ... @@ -1373,8 +1373,8 @@
1373 1373 return 1;
1374 1374 }
1375 1375  
1376   -static int fn_trie_lookup(struct fib_table *tb, const struct flowi *flp,
1377   - struct fib_result *res)
  1376 +int fib_table_lookup(struct fib_table *tb, const struct flowi *flp,
  1377 + struct fib_result *res)
1378 1378 {
1379 1379 struct trie *t = (struct trie *) tb->tb_data;
1380 1380 int ret;
... ... @@ -1595,7 +1595,7 @@
1595 1595 /*
1596 1596 * Caller must hold RTNL.
1597 1597 */
1598   -static int fn_trie_delete(struct fib_table *tb, struct fib_config *cfg)
  1598 +int fib_table_delete(struct fib_table *tb, struct fib_config *cfg)
1599 1599 {
1600 1600 struct trie *t = (struct trie *) tb->tb_data;
1601 1601 u32 key, mask;
... ... @@ -1786,7 +1786,7 @@
1786 1786 /*
1787 1787 * Caller must hold RTNL.
1788 1788 */
1789   -static int fn_trie_flush(struct fib_table *tb)
  1789 +int fib_table_flush(struct fib_table *tb)
1790 1790 {
1791 1791 struct trie *t = (struct trie *) tb->tb_data;
1792 1792 struct leaf *l, *ll = NULL;
... ... @@ -1807,9 +1807,9 @@
1807 1807 return found;
1808 1808 }
1809 1809  
1810   -static void fn_trie_select_default(struct fib_table *tb,
1811   - const struct flowi *flp,
1812   - struct fib_result *res)
  1810 +void fib_table_select_default(struct fib_table *tb,
  1811 + const struct flowi *flp,
  1812 + struct fib_result *res)
1813 1813 {
1814 1814 struct trie *t = (struct trie *) tb->tb_data;
1815 1815 int order, last_idx;
... ... @@ -1952,8 +1952,8 @@
1952 1952 return skb->len;
1953 1953 }
1954 1954  
1955   -static int fn_trie_dump(struct fib_table *tb, struct sk_buff *skb,
1956   - struct netlink_callback *cb)
  1955 +int fib_table_dump(struct fib_table *tb, struct sk_buff *skb,
  1956 + struct netlink_callback *cb)
1957 1957 {
1958 1958 struct leaf *l;
1959 1959 struct trie *t = (struct trie *) tb->tb_data;
... ... @@ -2020,12 +2020,6 @@
2020 2020  
2021 2021 tb->tb_id = id;
2022 2022 tb->tb_default = -1;
2023   - tb->tb_lookup = fn_trie_lookup;
2024   - tb->tb_insert = fn_trie_insert;
2025   - tb->tb_delete = fn_trie_delete;
2026   - tb->tb_flush = fn_trie_flush;
2027   - tb->tb_select_default = fn_trie_select_default;
2028   - tb->tb_dump = fn_trie_dump;
2029 2023  
2030 2024 t = (struct trie *) tb->tb_data;
2031 2025 memset(t, 0, sizeof(*t));