Improvements to the bitmap code

 * Removed all calls to drbd_bm_recount_bits()
 * add missing recv_cnt accounting for diskless read
 * more efficiently set large areas of bitmap
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

drivers/block/drbd/drbd_bitmap.c

@@ -401,29 +401,6 @@ static unsigned long bm_count_bits_swap_endian(struct drbd_bitmap *b)
 	return __bm_count_bits(b, 1, KM_USER0);
 }
 
-void _drbd_bm_recount_bits(struct drbd_conf *mdev, char *file, int line)
-{
-	struct drbd_bitmap *b = mdev->bitmap;
-	unsigned long flags, bits;
-
-	ERR_IF(!b) return;
-
-	/* IMO this should be inside drbd_bm_lock/unlock.
-	 * Unfortunately it is used outside of the locks.
-	 * And I'm not yet sure where we need to place the
-	 * lock/unlock correctly.
-	 */
-
-	spin_lock_irqsave(&b->bm_lock, flags);
-	bits = bm_count_bits(b);
-	if (bits != b->bm_set) {
-		dev_err(DEV, "bm_set was %lu, corrected to %lu. %s:%d\n",
-		    b->bm_set, bits, file, line);
-		b->bm_set = bits;
-	}
-	spin_unlock_irqrestore(&b->bm_lock, flags);
-}
-
 /* offset and len in long words.*/
 static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)
 {
@@ -1031,10 +1008,10 @@ unsigned long _drbd_bm_find_next_zero(struct drbd_conf *mdev, unsigned long bm_f
  * for val != 0, we change 0 -> 1, return code positiv
  * for val == 0, we change 1 -> 0, return code negative
  * wants bitnr, not sector.
+ * expected to be called for only a few bits (e - s about BITS_PER_LONG).
  * Must hold bitmap lock already. */
-
 int __bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
-	const unsigned long e, int val, const enum km_type km)
+	unsigned long e, int val, const enum km_type km)
 {
 	struct drbd_bitmap *b = mdev->bitmap;
 	unsigned long *p_addr = NULL;
@@ -1042,23 +1019,24 @@ int __bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
 	unsigned long last_page_nr = -1UL;
 	int c = 0;
 
+	if (e >= b->bm_bits) {
+		dev_err(DEV, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n",
+				s, e, b->bm_bits);
+		e = b->bm_bits ? b->bm_bits -1 : 0;
+	}
 	for (bitnr = s; bitnr <= e; bitnr++) {
-		ERR_IF (bitnr >= b->bm_bits) {
-			dev_err(DEV, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);
-		} else {
-			unsigned long offset = bitnr>>LN2_BPL;
-			unsigned long page_nr = offset >> (PAGE_SHIFT - LN2_BPL + 3);
-			if (page_nr != last_page_nr) {
-				if (p_addr)
-					__bm_unmap(p_addr, km);
-				p_addr = __bm_map_paddr(b, offset, km);
-				last_page_nr = page_nr;
-			}
-			if (val)
-				c += (0 == __test_and_set_bit(bitnr & BPP_MASK, p_addr));
-			else
-				c -= (0 != __test_and_clear_bit(bitnr & BPP_MASK, p_addr));
+		unsigned long offset = bitnr>>LN2_BPL;
+		unsigned long page_nr = offset >> (PAGE_SHIFT - LN2_BPL + 3);
+		if (page_nr != last_page_nr) {
+			if (p_addr)
+				__bm_unmap(p_addr, km);
+			p_addr = __bm_map_paddr(b, offset, km);
+			last_page_nr = page_nr;
 		}
+		if (val)
+			c += (0 == __test_and_set_bit(bitnr & BPP_MASK, p_addr));
+		else
+			c -= (0 != __test_and_clear_bit(bitnr & BPP_MASK, p_addr));
 	}
 	if (p_addr)
 		__bm_unmap(p_addr, km);
@@ -1102,12 +1080,73 @@ int drbd_bm_clear_bits(struct drbd_conf *mdev, const unsigned long s, const unsi
 	return -bm_change_bits_to(mdev, s, e, 0);
 }
 
+static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,
+		int page_nr, int first_word, int last_word)
+{
+	int i;
+	int bits;
+	unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr], KM_USER0);
+	for (i = first_word; i < last_word; i++) {
+		bits = hweight_long(paddr[i]);
+		paddr[i] = ~0UL;
+		b->bm_set += BITS_PER_LONG - bits;
+	}
+	kunmap_atomic(paddr, KM_USER0);
+}
+
 /* the same thing, but without taking the spin_lock_irqsave.
  * you must first drbd_bm_lock(). */
-int _drbd_bm_set_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
+void _drbd_bm_set_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
 {
-       /* WARN_ON(!bm_is_locked(b)); */
-       return __bm_change_bits_to(mdev, s, e, 1, KM_USER0);
+	/* s <= sl <= el <= e */
+	/* first set_bit from the first bit (s)
+	 * up to the next long boundary (sl),
+	 * then assign full words including the last long boundary (el),
+	 * then set_bit up to the last bit (e).
+	 * do not use memset, because we have need to account for changes,
+	 * so we need to loop over the words with hweight() anyways.
+	 */
+	unsigned long sl = ALIGN(s,BITS_PER_LONG);
+	unsigned long el = (e & ~((unsigned long)BITS_PER_LONG-1)) -1;
+	int first_page;
+	int last_page;
+	int page_nr;
+	int first_word;
+	int last_word;
+
+	if (e - s <= 3*BITS_PER_LONG) {
+		/* don't bother; el and sl may even be wrong. */
+		__bm_change_bits_to(mdev, s, e, 1, KM_USER0);
+		return;
+	}
+
+	/* difference is large enough that we can trust sl and el */
+
+	/* bits filling the current long */
+	if (sl)
+		__bm_change_bits_to(mdev, s, sl-1, 1, KM_USER0);
+
+	first_page = sl >> (3 + PAGE_SHIFT);
+	last_page = el >> (3 + PAGE_SHIFT);
+
+	/* MLPP: modulo longs per page */
+	/* LWPP: long words per page */
+	first_word = MLPP(sl >> LN2_BPL);
+	last_word = LWPP;
+
+	/* first and full pages, unless first page == last page */
+	for (page_nr = first_page; page_nr < last_page; page_nr++) {
+		bm_set_full_words_within_one_page(mdev->bitmap, page_nr, first_word, last_word);
+		cond_resched();
+		first_word = 0;
+	}
+
+	/* last page (respectively only page, for first page == last page) */
+	last_word = MLPP(el >> LN2_BPL);
+	bm_set_full_words_within_one_page(mdev->bitmap, last_page, first_word, last_word);
+
+	/* possibly trailing bits */
+	__bm_change_bits_to(mdev, el+1, e, 1, KM_USER0);
 }
 
 /* returns bit state

drivers/block/drbd/drbd_int.h

@@ -1306,7 +1306,7 @@ extern int  drbd_bm_set_bits(
 extern int  drbd_bm_clear_bits(
 		struct drbd_conf *mdev, unsigned long s, unsigned long e);
 /* bm_set_bits variant for use while holding drbd_bm_lock */
-extern int _drbd_bm_set_bits(struct drbd_conf *mdev,
+extern void _drbd_bm_set_bits(struct drbd_conf *mdev,
 		const unsigned long s, const unsigned long e);
 extern int  drbd_bm_test_bit(struct drbd_conf *mdev, unsigned long bitnr);
 extern int  drbd_bm_e_weight(struct drbd_conf *mdev, unsigned long enr);
@@ -1334,9 +1334,6 @@ extern void drbd_bm_get_lel(struct drbd_conf *mdev, size_t offset,
 extern void drbd_bm_lock(struct drbd_conf *mdev, char *why);
 extern void drbd_bm_unlock(struct drbd_conf *mdev);
 
-extern void _drbd_bm_recount_bits(struct drbd_conf *mdev, char *file, int line);
-#define drbd_bm_recount_bits(mdev) \
-	_drbd_bm_recount_bits(mdev, __FILE__, __LINE__)
 extern int drbd_bm_count_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e);
 /* drbd_main.c */
 

drivers/block/drbd/drbd_receiver.c

@@ -1381,6 +1381,10 @@ static int recv_dless_read(struct drbd_conf *mdev, struct drbd_request *req,
 
 	data_size -= dgs;
 
+	/* optimistically update recv_cnt.  if receiving fails below,
+	 * we disconnect anyways, and counters will be reset. */
+	mdev->recv_cnt += data_size>>9;
+
 	bio = req->master_bio;
 	D_ASSERT(sector == bio->bi_sector);
 
@@ -2484,8 +2488,6 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_conf *mdev, enum drbd_rol
 		}
 	}
 
-	drbd_bm_recount_bits(mdev);
-
 	return rv;
 }
 

drivers/block/drbd/drbd_worker.c

@@ -796,8 +796,6 @@ out:
 		drbd_queue_bitmap_io(mdev, &drbd_bm_write, NULL, "write from resync_finished");
 	}
 
-	drbd_bm_recount_bits(mdev);
-
 	if (khelper_cmd)
 		drbd_khelper(mdev, khelper_cmd);
 
@@ -1329,8 +1327,6 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
 	trace_drbd_resync(mdev, TRACE_LVL_SUMMARY, "Resync starting: side=%s\n",
 			  side == C_SYNC_TARGET ? "SyncTarget" : "SyncSource");
 
-	drbd_bm_recount_bits(mdev);
-
 	/* In case a previous resync run was aborted by an IO error/detach on the peer. */
 	drbd_rs_cancel_all(mdev);