Commit 16c6cf8bb471392fd09b48b7c27e7d83a446b4bc
Committed by
David S. Miller
1 parent
977750076d
Exists in
master
and in
7 other branches
ipv4: fib table algorithm performance improvement
The FIB algorithim for IPV4 is set at compile time, but kernel goes through the overhead of function call indirection at runtime. Save some cycles by turning the indirect calls to direct calls to either hash or trie code. Signed-off-by: Stephen Hemminger <shemminger@vyatta.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Showing 5 changed files with 48 additions and 56 deletions Side-by-side Diff
include/net/ip_fib.h
... | ... | @@ -144,18 +144,21 @@ |
144 | 144 | struct hlist_node tb_hlist; |
145 | 145 | u32 tb_id; |
146 | 146 | int tb_default; |
147 | - int (*tb_lookup)(struct fib_table *tb, const struct flowi *flp, struct fib_result *res); | |
148 | - int (*tb_insert)(struct fib_table *, struct fib_config *); | |
149 | - int (*tb_delete)(struct fib_table *, struct fib_config *); | |
150 | - int (*tb_dump)(struct fib_table *table, struct sk_buff *skb, | |
151 | - struct netlink_callback *cb); | |
152 | - int (*tb_flush)(struct fib_table *table); | |
153 | - void (*tb_select_default)(struct fib_table *table, | |
154 | - const struct flowi *flp, struct fib_result *res); | |
155 | - | |
156 | 147 | unsigned char tb_data[0]; |
157 | 148 | }; |
158 | 149 | |
150 | +extern int fib_table_lookup(struct fib_table *tb, const struct flowi *flp, | |
151 | + struct fib_result *res); | |
152 | +extern int fib_table_insert(struct fib_table *, struct fib_config *); | |
153 | +extern int fib_table_delete(struct fib_table *, struct fib_config *); | |
154 | +extern int fib_table_dump(struct fib_table *table, struct sk_buff *skb, | |
155 | + struct netlink_callback *cb); | |
156 | +extern int fib_table_flush(struct fib_table *table); | |
157 | +extern void fib_table_select_default(struct fib_table *table, | |
158 | + const struct flowi *flp, | |
159 | + struct fib_result *res); | |
160 | + | |
161 | + | |
159 | 162 | #ifndef CONFIG_IP_MULTIPLE_TABLES |
160 | 163 | |
161 | 164 | #define TABLE_LOCAL_INDEX 0 |
162 | 165 | |
... | ... | @@ -182,11 +185,11 @@ |
182 | 185 | struct fib_table *table; |
183 | 186 | |
184 | 187 | table = fib_get_table(net, RT_TABLE_LOCAL); |
185 | - if (!table->tb_lookup(table, flp, res)) | |
188 | + if (!fib_table_lookup(table, flp, res)) | |
186 | 189 | return 0; |
187 | 190 | |
188 | 191 | table = fib_get_table(net, RT_TABLE_MAIN); |
189 | - if (!table->tb_lookup(table, flp, res)) | |
192 | + if (!fib_table_lookup(table, flp, res)) | |
190 | 193 | return 0; |
191 | 194 | return -ENETUNREACH; |
192 | 195 | } |
net/ipv4/fib_frontend.c
... | ... | @@ -125,7 +125,7 @@ |
125 | 125 | #endif |
126 | 126 | tb = fib_get_table(net, table); |
127 | 127 | if (FIB_RES_GW(*res) && FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK) |
128 | - tb->tb_select_default(tb, flp, res); | |
128 | + fib_table_select_default(tb, flp, res); | |
129 | 129 | } |
130 | 130 | |
131 | 131 | static void fib_flush(struct net *net) |
... | ... | @@ -139,7 +139,7 @@ |
139 | 139 | for (h = 0; h < FIB_TABLE_HASHSZ; h++) { |
140 | 140 | head = &net->ipv4.fib_table_hash[h]; |
141 | 141 | hlist_for_each_entry(tb, node, head, tb_hlist) |
142 | - flushed += tb->tb_flush(tb); | |
142 | + flushed += fib_table_flush(tb); | |
143 | 143 | } |
144 | 144 | |
145 | 145 | if (flushed) |
... | ... | @@ -162,7 +162,7 @@ |
162 | 162 | #endif |
163 | 163 | |
164 | 164 | local_table = fib_get_table(net, RT_TABLE_LOCAL); |
165 | - if (!local_table || local_table->tb_lookup(local_table, &fl, &res)) | |
165 | + if (!local_table || fib_table_lookup(local_table, &fl, &res)) | |
166 | 166 | return NULL; |
167 | 167 | if (res.type != RTN_LOCAL) |
168 | 168 | goto out; |
... | ... | @@ -200,7 +200,7 @@ |
200 | 200 | local_table = fib_get_table(net, RT_TABLE_LOCAL); |
201 | 201 | if (local_table) { |
202 | 202 | ret = RTN_UNICAST; |
203 | - if (!local_table->tb_lookup(local_table, &fl, &res)) { | |
203 | + if (!fib_table_lookup(local_table, &fl, &res)) { | |
204 | 204 | if (!dev || dev == res.fi->fib_dev) |
205 | 205 | ret = res.type; |
206 | 206 | fib_res_put(&res); |
207 | 207 | |
... | ... | @@ -473,13 +473,13 @@ |
473 | 473 | if (cmd == SIOCDELRT) { |
474 | 474 | tb = fib_get_table(net, cfg.fc_table); |
475 | 475 | if (tb) |
476 | - err = tb->tb_delete(tb, &cfg); | |
476 | + err = fib_table_delete(tb, &cfg); | |
477 | 477 | else |
478 | 478 | err = -ESRCH; |
479 | 479 | } else { |
480 | 480 | tb = fib_new_table(net, cfg.fc_table); |
481 | 481 | if (tb) |
482 | - err = tb->tb_insert(tb, &cfg); | |
482 | + err = fib_table_insert(tb, &cfg); | |
483 | 483 | else |
484 | 484 | err = -ENOBUFS; |
485 | 485 | } |
... | ... | @@ -594,7 +594,7 @@ |
594 | 594 | goto errout; |
595 | 595 | } |
596 | 596 | |
597 | - err = tb->tb_delete(tb, &cfg); | |
597 | + err = fib_table_delete(tb, &cfg); | |
598 | 598 | errout: |
599 | 599 | return err; |
600 | 600 | } |
... | ... | @@ -616,7 +616,7 @@ |
616 | 616 | goto errout; |
617 | 617 | } |
618 | 618 | |
619 | - err = tb->tb_insert(tb, &cfg); | |
619 | + err = fib_table_insert(tb, &cfg); | |
620 | 620 | errout: |
621 | 621 | return err; |
622 | 622 | } |
... | ... | @@ -647,7 +647,7 @@ |
647 | 647 | if (dumped) |
648 | 648 | memset(&cb->args[2], 0, sizeof(cb->args) - |
649 | 649 | 2 * sizeof(cb->args[0])); |
650 | - if (tb->tb_dump(tb, skb, cb) < 0) | |
650 | + if (fib_table_dump(tb, skb, cb) < 0) | |
651 | 651 | goto out; |
652 | 652 | dumped = 1; |
653 | 653 | next: |
654 | 654 | |
... | ... | @@ -701,9 +701,9 @@ |
701 | 701 | cfg.fc_scope = RT_SCOPE_HOST; |
702 | 702 | |
703 | 703 | if (cmd == RTM_NEWROUTE) |
704 | - tb->tb_insert(tb, &cfg); | |
704 | + fib_table_insert(tb, &cfg); | |
705 | 705 | else |
706 | - tb->tb_delete(tb, &cfg); | |
706 | + fib_table_delete(tb, &cfg); | |
707 | 707 | } |
708 | 708 | |
709 | 709 | void fib_add_ifaddr(struct in_ifaddr *ifa) |
... | ... | @@ -832,7 +832,7 @@ |
832 | 832 | local_bh_disable(); |
833 | 833 | |
834 | 834 | frn->tb_id = tb->tb_id; |
835 | - frn->err = tb->tb_lookup(tb, &fl, &res); | |
835 | + frn->err = fib_table_lookup(tb, &fl, &res); | |
836 | 836 | |
837 | 837 | if (!frn->err) { |
838 | 838 | frn->prefixlen = res.prefixlen; |
... | ... | @@ -1009,7 +1009,7 @@ |
1009 | 1009 | head = &net->ipv4.fib_table_hash[i]; |
1010 | 1010 | hlist_for_each_entry_safe(tb, node, tmp, head, tb_hlist) { |
1011 | 1011 | hlist_del(node); |
1012 | - tb->tb_flush(tb); | |
1012 | + fib_table_flush(tb); | |
1013 | 1013 | kfree(tb); |
1014 | 1014 | } |
1015 | 1015 | } |
net/ipv4/fib_hash.c
... | ... | @@ -242,8 +242,8 @@ |
242 | 242 | return fz; |
243 | 243 | } |
244 | 244 | |
245 | -static int | |
246 | -fn_hash_lookup(struct fib_table *tb, const struct flowi *flp, struct fib_result *res) | |
245 | +int fib_table_lookup(struct fib_table *tb, | |
246 | + const struct flowi *flp, struct fib_result *res) | |
247 | 247 | { |
248 | 248 | int err; |
249 | 249 | struct fn_zone *fz; |
... | ... | @@ -274,8 +274,8 @@ |
274 | 274 | return err; |
275 | 275 | } |
276 | 276 | |
277 | -static void | |
278 | -fn_hash_select_default(struct fib_table *tb, const struct flowi *flp, struct fib_result *res) | |
277 | +void fib_table_select_default(struct fib_table *tb, | |
278 | + const struct flowi *flp, struct fib_result *res) | |
279 | 279 | { |
280 | 280 | int order, last_idx; |
281 | 281 | struct hlist_node *node; |
... | ... | @@ -366,7 +366,7 @@ |
366 | 366 | return NULL; |
367 | 367 | } |
368 | 368 | |
369 | -static int fn_hash_insert(struct fib_table *tb, struct fib_config *cfg) | |
369 | +int fib_table_insert(struct fib_table *tb, struct fib_config *cfg) | |
370 | 370 | { |
371 | 371 | struct fn_hash *table = (struct fn_hash *) tb->tb_data; |
372 | 372 | struct fib_node *new_f = NULL; |
... | ... | @@ -544,8 +544,7 @@ |
544 | 544 | return err; |
545 | 545 | } |
546 | 546 | |
547 | - | |
548 | -static int fn_hash_delete(struct fib_table *tb, struct fib_config *cfg) | |
547 | +int fib_table_delete(struct fib_table *tb, struct fib_config *cfg) | |
549 | 548 | { |
550 | 549 | struct fn_hash *table = (struct fn_hash *)tb->tb_data; |
551 | 550 | struct fib_node *f; |
... | ... | @@ -662,7 +661,7 @@ |
662 | 661 | return found; |
663 | 662 | } |
664 | 663 | |
665 | -static int fn_hash_flush(struct fib_table *tb) | |
664 | +int fib_table_flush(struct fib_table *tb) | |
666 | 665 | { |
667 | 666 | struct fn_hash *table = (struct fn_hash *) tb->tb_data; |
668 | 667 | struct fn_zone *fz; |
... | ... | @@ -743,7 +742,8 @@ |
743 | 742 | return skb->len; |
744 | 743 | } |
745 | 744 | |
746 | -static int fn_hash_dump(struct fib_table *tb, struct sk_buff *skb, struct netlink_callback *cb) | |
745 | +int fib_table_dump(struct fib_table *tb, struct sk_buff *skb, | |
746 | + struct netlink_callback *cb) | |
747 | 747 | { |
748 | 748 | int m, s_m; |
749 | 749 | struct fn_zone *fz; |
... | ... | @@ -787,12 +787,7 @@ |
787 | 787 | |
788 | 788 | tb->tb_id = id; |
789 | 789 | tb->tb_default = -1; |
790 | - tb->tb_lookup = fn_hash_lookup; | |
791 | - tb->tb_insert = fn_hash_insert; | |
792 | - tb->tb_delete = fn_hash_delete; | |
793 | - tb->tb_flush = fn_hash_flush; | |
794 | - tb->tb_select_default = fn_hash_select_default; | |
795 | - tb->tb_dump = fn_hash_dump; | |
790 | + | |
796 | 791 | memset(tb->tb_data, 0, sizeof(struct fn_hash)); |
797 | 792 | return tb; |
798 | 793 | } |
net/ipv4/fib_rules.c
... | ... | @@ -94,7 +94,7 @@ |
94 | 94 | if ((tbl = fib_get_table(rule->fr_net, rule->table)) == NULL) |
95 | 95 | goto errout; |
96 | 96 | |
97 | - err = tbl->tb_lookup(tbl, flp, (struct fib_result *) arg->result); | |
97 | + err = fib_table_lookup(tbl, flp, (struct fib_result *) arg->result); | |
98 | 98 | if (err > 0) |
99 | 99 | err = -EAGAIN; |
100 | 100 | errout: |
net/ipv4/fib_trie.c
... | ... | @@ -1174,7 +1174,7 @@ |
1174 | 1174 | /* |
1175 | 1175 | * Caller must hold RTNL. |
1176 | 1176 | */ |
1177 | -static int fn_trie_insert(struct fib_table *tb, struct fib_config *cfg) | |
1177 | +int fib_table_insert(struct fib_table *tb, struct fib_config *cfg) | |
1178 | 1178 | { |
1179 | 1179 | struct trie *t = (struct trie *) tb->tb_data; |
1180 | 1180 | struct fib_alias *fa, *new_fa; |
... | ... | @@ -1373,8 +1373,8 @@ |
1373 | 1373 | return 1; |
1374 | 1374 | } |
1375 | 1375 | |
1376 | -static int fn_trie_lookup(struct fib_table *tb, const struct flowi *flp, | |
1377 | - struct fib_result *res) | |
1376 | +int fib_table_lookup(struct fib_table *tb, const struct flowi *flp, | |
1377 | + struct fib_result *res) | |
1378 | 1378 | { |
1379 | 1379 | struct trie *t = (struct trie *) tb->tb_data; |
1380 | 1380 | int ret; |
... | ... | @@ -1595,7 +1595,7 @@ |
1595 | 1595 | /* |
1596 | 1596 | * Caller must hold RTNL. |
1597 | 1597 | */ |
1598 | -static int fn_trie_delete(struct fib_table *tb, struct fib_config *cfg) | |
1598 | +int fib_table_delete(struct fib_table *tb, struct fib_config *cfg) | |
1599 | 1599 | { |
1600 | 1600 | struct trie *t = (struct trie *) tb->tb_data; |
1601 | 1601 | u32 key, mask; |
... | ... | @@ -1786,7 +1786,7 @@ |
1786 | 1786 | /* |
1787 | 1787 | * Caller must hold RTNL. |
1788 | 1788 | */ |
1789 | -static int fn_trie_flush(struct fib_table *tb) | |
1789 | +int fib_table_flush(struct fib_table *tb) | |
1790 | 1790 | { |
1791 | 1791 | struct trie *t = (struct trie *) tb->tb_data; |
1792 | 1792 | struct leaf *l, *ll = NULL; |
... | ... | @@ -1807,9 +1807,9 @@ |
1807 | 1807 | return found; |
1808 | 1808 | } |
1809 | 1809 | |
1810 | -static void fn_trie_select_default(struct fib_table *tb, | |
1811 | - const struct flowi *flp, | |
1812 | - struct fib_result *res) | |
1810 | +void fib_table_select_default(struct fib_table *tb, | |
1811 | + const struct flowi *flp, | |
1812 | + struct fib_result *res) | |
1813 | 1813 | { |
1814 | 1814 | struct trie *t = (struct trie *) tb->tb_data; |
1815 | 1815 | int order, last_idx; |
... | ... | @@ -1952,8 +1952,8 @@ |
1952 | 1952 | return skb->len; |
1953 | 1953 | } |
1954 | 1954 | |
1955 | -static int fn_trie_dump(struct fib_table *tb, struct sk_buff *skb, | |
1956 | - struct netlink_callback *cb) | |
1955 | +int fib_table_dump(struct fib_table *tb, struct sk_buff *skb, | |
1956 | + struct netlink_callback *cb) | |
1957 | 1957 | { |
1958 | 1958 | struct leaf *l; |
1959 | 1959 | struct trie *t = (struct trie *) tb->tb_data; |
... | ... | @@ -2020,12 +2020,6 @@ |
2020 | 2020 | |
2021 | 2021 | tb->tb_id = id; |
2022 | 2022 | tb->tb_default = -1; |
2023 | - tb->tb_lookup = fn_trie_lookup; | |
2024 | - tb->tb_insert = fn_trie_insert; | |
2025 | - tb->tb_delete = fn_trie_delete; | |
2026 | - tb->tb_flush = fn_trie_flush; | |
2027 | - tb->tb_select_default = fn_trie_select_default; | |
2028 | - tb->tb_dump = fn_trie_dump; | |
2029 | 2023 | |
2030 | 2024 | t = (struct trie *) tb->tb_data; |
2031 | 2025 | memset(t, 0, sizeof(*t)); |