PCI: iova RB tree setup tweak

The following patch merges two functions into one allowing for a 3% reduction in overhead in locating, allocating and inserting pages for use in IOMMU operations. Its a bit of a eye-crosser so I welcome any RB-tree / MM experts to take a look. It works by re-using some of the information gathered in the search for the pages to use in setting up the IOTLB's in the insertion of the iova structure into the RB tree. Signed-off-by: <mgross@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

PCI: iova RB tree setup tweak
The following patch merges two functions into one allowing for a 3% reduction in overhead in locating, allocating and inserting pages for use in IOMMU operations. Its a bit of a eye-crosser so I welcome any RB-tree / MM experts to take a look. It works by re-using some of the information gathered in the search for the pages to use in setting up the IOTLB's in the insertion of the iova structure into the RB tree. Signed-off-by: <mgross@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
mark gross · Greg Kroah-Hartman
1 parent c9e9e0bfc5
Showing 1 changed file with 36 additions and 14 deletions Side-by-side Diff
drivers/pci/iova.c
@@ -73,10 +73,11 @@
 	return pad_size;
 }
  
-static int __alloc_iova_range(struct iova_domain *iovad, unsigned long size,
-		unsigned long limit_pfn, struct iova *new, bool size_aligned)
+static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
+		unsigned long size, unsigned long limit_pfn,
+			struct iova *new, bool size_aligned)
 {
-	struct rb_node *curr = NULL;
+	struct rb_node *prev, *curr = NULL;
 	unsigned long flags;
 	unsigned long saved_pfn;
 	unsigned int pad_size = 0;
  
@@ -85,8 +86,10 @@
 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
 	saved_pfn = limit_pfn;
 	curr = __get_cached_rbnode(iovad, &limit_pfn);
+	prev = curr;
 	while (curr) {
 		struct iova *curr_iova = container_of(curr, struct iova, node);
+
 		if (limit_pfn < curr_iova->pfn_lo)
 			goto move_left;
 		else if (limit_pfn < curr_iova->pfn_hi)
@@ -100,6 +103,7 @@
 adjust_limit_pfn:
 		limit_pfn = curr_iova->pfn_lo - 1;
 move_left:
+		prev = curr;
 		curr = rb_prev(curr);
 	}
  
  
@@ -116,7 +120,33 @@
 	new->pfn_lo = limit_pfn - (size + pad_size) + 1;
 	new->pfn_hi = new->pfn_lo + size - 1;
  
+	/* Insert the new_iova into domain rbtree by holding writer lock */
+	/* Add new node and rebalance tree. */
+	{
+		struct rb_node **entry = &((prev)), *parent = NULL;
+		/* Figure out where to put new node */
+		while (*entry) {
+			struct iova *this = container_of(*entry,
+							struct iova, node);
+			parent = *entry;
+
+			if (new->pfn_lo < this->pfn_lo)
+				entry = &((*entry)->rb_left);
+			else if (new->pfn_lo > this->pfn_lo)
+				entry = &((*entry)->rb_right);
+			else
+				BUG(); /* this should not happen */
+		}
+
+		/* Add new node and rebalance tree. */
+		rb_link_node(&new->node, parent, entry);
+		rb_insert_color(&new->node, &iovad->rbroot);
+	}
+	__cached_rbnode_insert_update(iovad, saved_pfn, new);
+
 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
+
+
 	return 0;
 }
  
  
  
  
@@ -172,22 +202,14 @@
 		size = __roundup_pow_of_two(size);
  
 	spin_lock_irqsave(&iovad->iova_alloc_lock, flags);
-	ret = __alloc_iova_range(iovad, size, limit_pfn, new_iova,
-			size_aligned);
+	ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,
+			new_iova, size_aligned);
  
+	spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
 	if (ret) {
-		spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
 		free_iova_mem(new_iova);
 		return NULL;
 	}
-
-	/* Insert the new_iova into domain rbtree by holding writer lock */
-	spin_lock(&iovad->iova_rbtree_lock);
-	iova_insert_rbtree(&iovad->rbroot, new_iova);
-	__cached_rbnode_insert_update(iovad, limit_pfn, new_iova);
-	spin_unlock(&iovad->iova_rbtree_lock);
-
-	spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
  
 	return new_iova;
 }
...	...	@@ -73,10 +73,11 @@
73	73	return pad_size;
74	74	}
75	75
76		-static int __alloc_iova_range(struct iova_domain *iovad, unsigned long size,
77		- unsigned long limit_pfn, struct iova *new, bool size_aligned)
	76	+static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
	77	+ unsigned long size, unsigned long limit_pfn,
	78	+ struct iova *new, bool size_aligned)
78	79	{
79		- struct rb_node *curr = NULL;
	80	+ struct rb_node prev, curr = NULL;
80	81	unsigned long flags;
81	82	unsigned long saved_pfn;
82	83	unsigned int pad_size = 0;
83	84
...	...	@@ -85,8 +86,10 @@
85	86	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
86	87	saved_pfn = limit_pfn;
87	88	curr = __get_cached_rbnode(iovad, &limit_pfn);
	89	+ prev = curr;
88	90	while (curr) {
89	91	struct iova *curr_iova = container_of(curr, struct iova, node);
	92	+
90	93	if (limit_pfn < curr_iova->pfn_lo)
91	94	goto move_left;
92	95	else if (limit_pfn < curr_iova->pfn_hi)
...	...	@@ -100,6 +103,7 @@
100	103	adjust_limit_pfn:
101	104	limit_pfn = curr_iova->pfn_lo - 1;
102	105	move_left:
	106	+ prev = curr;
103	107	curr = rb_prev(curr);
104	108	}
105	109
106	110
...	...	@@ -116,7 +120,33 @@
116	120	new->pfn_lo = limit_pfn - (size + pad_size) + 1;
117	121	new->pfn_hi = new->pfn_lo + size - 1;
118	122
	123	+ /* Insert the new_iova into domain rbtree by holding writer lock */
	124	+ /* Add new node and rebalance tree. */
	125	+ {
	126	+ struct rb_node *entry = &((prev)), parent = NULL;
	127	+ /* Figure out where to put new node */
	128	+ while (*entry) {
	129	+ struct iova this = container_of(entry,
	130	+ struct iova, node);
	131	+ parent = *entry;
	132	+
	133	+ if (new->pfn_lo < this->pfn_lo)
	134	+ entry = &((*entry)->rb_left);
	135	+ else if (new->pfn_lo > this->pfn_lo)
	136	+ entry = &((*entry)->rb_right);
	137	+ else
	138	+ BUG(); /* this should not happen */
	139	+ }
	140	+
	141	+ /* Add new node and rebalance tree. */
	142	+ rb_link_node(&new->node, parent, entry);
	143	+ rb_insert_color(&new->node, &iovad->rbroot);
	144	+ }
	145	+ __cached_rbnode_insert_update(iovad, saved_pfn, new);
	146	+
119	147	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
	148	+
	149	+
120	150	return 0;
121	151	}
122	152
123	153
124	154
125	155
...	...	@@ -172,22 +202,14 @@
172	202	size = __roundup_pow_of_two(size);
173	203
174	204	spin_lock_irqsave(&iovad->iova_alloc_lock, flags);
175		- ret = __alloc_iova_range(iovad, size, limit_pfn, new_iova,
176		- size_aligned);
	205	+ ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,
	206	+ new_iova, size_aligned);
177	207
	208	+ spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
178	209	if (ret) {
179		- spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
180	210	free_iova_mem(new_iova);
181	211	return NULL;
182	212	}
183		-
184		- /* Insert the new_iova into domain rbtree by holding writer lock */
185		- spin_lock(&iovad->iova_rbtree_lock);
186		- iova_insert_rbtree(&iovad->rbroot, new_iova);
187		- __cached_rbnode_insert_update(iovad, limit_pfn, new_iova);
188		- spin_unlock(&iovad->iova_rbtree_lock);
189		-
190		- spin_unlock_irqrestore(&iovad->iova_alloc_lock, flags);
191	213
192	214	return new_iova;
193	215	}