# -*- coding: utf-8 -*- """ 业务中经常需要对redis执行一组连续的gitbit 以下通过getrange 方法来优化getbits的速度 empty key, times 10 git_bits 1: 0.508 get_bits_pipe 1: 0.604 git_bits 10: 0.543 get_bits_pipe 10: 1.159 git_bits 100: 0.587 get_bits_pipe 100: 3.477 git_bits 1000: 0.744 get_bits_pipe 1000: 34.866 git_bits 5000: 0.629 get_bits_pipe 5000: 120.348 all true, times 10 git_bits 1: 0.405 get_bits_pipe 1: 0.478 git_bits 10: 0.420 get_bits_pipe 10: 1.390 git_bits 100: 0.995 get_bits_pipe 100: 3.562 git_bits 1000: 1.615 get_bits_pipe 1000: 28.776 git_bits 5000: 4.770 get_bits_pipe 5000: 120.539 由此可见getrange 确实有效的提升了查询性能 """ import time import redis redis_bit_map = { "\x00": [], "\x01": [7], "\x02": [6], "\x03": [6, 7], "\x04": [5], "\x05": [5, 7], "\x06": [5, 6], "\x07": [5, 6, 7], "\x08": [4], "\t": [4, 7], "\n": [4, 6], "\x0b": [4, 6, 7], "\x0c": [4, 5], "\r": [4, 5, 7], "\x0e": [4, 5, 6], "\x0f": [4, 5, 6, 7], "\x10": [3], "\x11": [3, 7], "\x12": [3, 6], "\x13": [3, 6, 7], "\x14": [3, 5], "\x15": [3, 5, 7], "\x16": [3, 5, 6], "\x17": [3, 5, 6, 7], "\x18": [3, 4], "\x19": [3, 4, 7], "\x1a": [3, 4, 6], "\x1b": [3, 4, 6, 7], "\x1c": [3, 4, 5], "\x1d": [3, 4, 5, 7], "\x1e": [3, 4, 5, 6], "\x1f": [3, 4, 5, 6, 7], " ": [2], "!": [2, 7], """: [2, 6], "#": [2, 6, 7], "$": [2, 5], "%": [2, 5, 7], "&": [2, 5, 6], """: [2, 5, 6, 7], "(": [2, 4], ")": [2, 4, 7], "*": [2, 4, 6], "+": [2, 4, 6, 7], ",": [2, 4, 5], "-": [2, 4, 5, 7], ".": [2, 4, 5, 6], "/": [2, 4, 5, 6, 7], "0": [2, 3], "1": [2, 3, 7], "2": [2, 3, 6], "3": [2, 3, 6, 7], "4": [2, 3, 5], "5": [2, 3, 5, 7], "6": [2, 3, 5, 6], "7": [2, 3, 5, 6, 7], "8": [2, 3, 4], "9": [2, 3, 4, 7], ":": [2, 3, 4, 6], ";": [2, 3, 4, 6, 7], "<": [2, 3, 4, 5], "=": [2, 3, 4, 5, 7], ">": [2, 3, 4, 5, 6], "?": [2, 3, 4, 5, 6, 7], "@": [1], "A": [1, 7], "B": [1, 6], "C": [1, 6, 7], "D": [1, 5], "E": [1, 5, 7], "F": [1, 5, 6], "G": [1, 5, 6, 7], "H": [1, 4], "I": [1, 4, 7], "J": [1, 4, 6], "K": [1, 4, 6, 7], "L": [1, 4, 5], "M": [1, 4, 5, 7], "N": [1, 4, 5, 6], "O": [1, 4, 5, 6, 7], "P": [1, 3], "Q": [1, 3, 7], "R": [1, 3, 6], "S": [1, 3, 6, 7], "T": [1, 3, 5], "U": [1, 3, 5, 7], "V": [1, 3, 5, 6], "W": [1, 3, 5, 6, 7], "X": [1, 3, 4], "Y": [1, 3, 4, 7], "Z": [1, 3, 4, 6], "[": [1, 3, 4, 6, 7], "\\": [1, 3, 4, 5], "]": [1, 3, 4, 5, 7], "^": [1, 3, 4, 5, 6], "_": [1, 3, 4, 5, 6, 7], "`": [1, 2], "a": [1, 2, 7], "b": [1, 2, 6], "c": [1, 2, 6, 7], "d": [1, 2, 5], "e": [1, 2, 5, 7], "f": [1, 2, 5, 6], "g": [1, 2, 5, 6, 7], "h": [1, 2, 4], "i": [1, 2, 4, 7], "j": [1, 2, 4, 6], "k": [1, 2, 4, 6, 7], "l": [1, 2, 4, 5], "m": [1, 2, 4, 5, 7], "n": [1, 2, 4, 5, 6], "o": [1, 2, 4, 5, 6, 7], "p": [1, 2, 3], "q": [1, 2, 3, 7], "r": [1, 2, 3, 6], "s": [1, 2, 3, 6, 7], "t": [1, 2, 3, 5], "u": [1, 2, 3, 5, 7], "v": [1, 2, 3, 5, 6], "w": [1, 2, 3, 5, 6, 7], "x": [1, 2, 3, 4], "y": [1, 2, 3, 4, 7], "z": [1, 2, 3, 4, 6], "{": [1, 2, 3, 4, 6, 7], "|": [1, 2, 3, 4, 5], "}": [1, 2, 3, 4, 5, 7], "~": [1, 2, 3, 4, 5, 6], "\x7f": [1, 2, 3, 4, 5, 6, 7], "\x80": [0], "\x81": [0, 7], "\x82": [0, 6], "\x83": [0, 6, 7], "\x84": [0, 5], "\x85": [0, 5, 7], "\x86": [0, 5, 6], "\x87": [0, 5, 6, 7], "\x88": [0, 4], "\x89": [0, 4, 7], "\x8a": [0, 4, 6], "\x8b": [0, 4, 6, 7], "\x8c": [0, 4, 5], "\x8d": [0, 4, 5, 7], "\x8e": [0, 4, 5, 6], "\x8f": [0, 4, 5, 6, 7], "\x90": [0, 3], "\x91": [0, 3, 7], "\x92": [0, 3, 6], "\x93": [0, 3, 6, 7], "\x94": [0, 3, 5], "\x95": [0, 3, 5, 7], "\x96": [0, 3, 5, 6], "\x97": [0, 3, 5, 6, 7], "\x98": [0, 3, 4], "\x99": [0, 3, 4, 7], "\x9a": [0, 3, 4, 6], "\x9b": [0, 3, 4, 6, 7], "\x9c": [0, 3, 4, 5], "\x9d": [0, 3, 4, 5, 7], "\x9e": [0, 3, 4, 5, 6], "\x9f": [0, 3, 4, 5, 6, 7], "\xa0": [0, 2], "\xa1": [0, 2, 7], "\xa2": [0, 2, 6], "\xa3": [0, 2, 6, 7], "\xa4": [0, 2, 5], "\xa5": [0, 2, 5, 7], "\xa6": [0, 2, 5, 6], "\xa7": [0, 2, 5, 6, 7], "\xa8": [0, 2, 4], "\xa9": [0, 2, 4, 7], "\xaa": [0, 2, 4, 6], "\xab": [0, 2, 4, 6, 7], "\xac": [0, 2, 4, 5], "\xad": [0, 2, 4, 5, 7], "\xae": [0, 2, 4, 5, 6], "\xaf": [0, 2, 4, 5, 6, 7], "\xb0": [0, 2, 3], "\xb1": [0, 2, 3, 7], "\xb2": [0, 2, 3, 6], "\xb3": [0, 2, 3, 6, 7], "\xb4": [0, 2, 3, 5], "\xb5": [0, 2, 3, 5, 7], "\xb6": [0, 2, 3, 5, 6], "\xb7": [0, 2, 3, 5, 6, 7], "\xb8": [0, 2, 3, 4], "\xb9": [0, 2, 3, 4, 7], "\xba": [0, 2, 3, 4, 6], "\xbb": [0, 2, 3, 4, 6, 7], "\xbc": [0, 2, 3, 4, 5], "\xbd": [0, 2, 3, 4, 5, 7], "\xbe": [0, 2, 3, 4, 5, 6], "\xbf": [0, 2, 3, 4, 5, 6, 7], "\xc0": [0, 1], "\xc1": [0, 1, 7], "\xc2": [0, 1, 6], "\xc3": [0, 1, 6, 7], "\xc4": [0, 1, 5], "\xc5": [0, 1, 5, 7], "\xc6": [0, 1, 5, 6], "\xc7": [0, 1, 5, 6, 7], "\xc8": [0, 1, 4], "\xc9": [0, 1, 4, 7], "\xca": [0, 1, 4, 6], "\xcb": [0, 1, 4, 6, 7], "\xcc": [0, 1, 4, 5], "\xcd": [0, 1, 4, 5, 7], "\xce": [0, 1, 4, 5, 6], "\xcf": [0, 1, 4, 5, 6, 7], "\xd0": [0, 1, 3], "\xd1": [0, 1, 3, 7], "\xd2": [0, 1, 3, 6], "\xd3": [0, 1, 3, 6, 7], "\xd4": [0, 1, 3, 5], "\xd5": [0, 1, 3, 5, 7], "\xd6": [0, 1, 3, 5, 6], "\xd7": [0, 1, 3, 5, 6, 7], "\xd8": [0, 1, 3, 4], "\xd9": [0, 1, 3, 4, 7], "\xda": [0, 1, 3, 4, 6], "\xdb": [0, 1, 3, 4, 6, 7], "\xdc": [0, 1, 3, 4, 5], "\xdd": [0, 1, 3, 4, 5, 7], "\xde": [0, 1, 3, 4, 5, 6], "\xdf": [0, 1, 3, 4, 5, 6, 7], "\xe0": [0, 1, 2], "\xe1": [0, 1, 2, 7], "\xe2": [0, 1, 2, 6], "\xe3": [0, 1, 2, 6, 7], "\xe4": [0, 1, 2, 5], "\xe5": [0, 1, 2, 5, 7], "\xe6": [0, 1, 2, 5, 6], "\xe7": [0, 1, 2, 5, 6, 7], "\xe8": [0, 1, 2, 4], "\xe9": [0, 1, 2, 4, 7], "\xea": [0, 1, 2, 4, 6], "\xeb": [0, 1, 2, 4, 6, 7], "\xec": [0, 1, 2, 4, 5], "\xed": [0, 1, 2, 4, 5, 7], "\xee": [0, 1, 2, 4, 5, 6], "\xef": [0, 1, 2, 4, 5, 6, 7], "\xf0": [0, 1, 2, 3], "\xf1": [0, 1, 2, 3, 7], "\xf2": [0, 1, 2, 3, 6], "\xf3": [0, 1, 2, 3, 6, 7], "\xf4": [0, 1, 2, 3, 5], "\xf5": [0, 1, 2, 3, 5, 7], "\xf6": [0, 1, 2, 3, 5, 6], "\xf7": [0, 1, 2, 3, 5, 6, 7], "\xf8": [0, 1, 2, 3, 4], "\xf9": [0, 1, 2, 3, 4, 7], "\xfa": [0, 1, 2, 3, 4, 6], "\xfb": [0, 1, 2, 3, 4, 6, 7], "\xfc": [0, 1, 2, 3, 4, 5], "\xfd": [0, 1, 2, 3, 4, 5, 7], "\xfe": [0, 1, 2, 3, 4, 5, 6], "\xff": [0, 1, 2, 3, 4, 5, 6, 7], } def find_bit(bits): """ 位计算为真的下标 >>> find_bit(b"\x80") [0] >>> find_bit("@") [1] >>> find_bit(b"\xff\xff") [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] >>> find_bit(b"\x11") [3, 7] >>> find_bit(b"\x01\x01") [7, 15] >>> find_bit(b"\x11\x11") [3, 7, 11, 15] """ return [no * 8 + j for no, i in enumerate(bits) for j in redis_bit_map[i]] def git_bits(redis_client, key, cur, num=10): """ 按位查询 返回bit位为1的下标 :param cur: 开始位置 :param num: 查询个数 :return: [] """ start, offset = divmod(cur, 8) # end 必须按8取整 end = (cur + num + 7) / 8 return [start * 8 + _id for _id in find_bit(redis_client.getrange(key, start, end)) if offset <= _id <= offset + num] def get_bits_pipe(pipe, key, cur, num=10): """ 使用pipeline gitbit""" for i in xrange(num + 1): pipe.getbit(key, cur + i) return [cur + i for i, v in enumerate(pipe.execute()) if v] def bench(func, *a, **kw): """压力测试""" s = time.time() times = kw.pop("times", 1) res = None for _ in range(times): res = func(*a, **kw) e = time.time() print("%s %d: %.3f" % (func.func_name, kw["num"], (e - s) * 1000 / times)) if __name__ == "__main__": key = "bench_bits" r = redis.Redis() Times = 10 print("empty key, times %d" % Times) r.delete(key) pipe = r.pipeline(transaction=False) for MAX in (1, 10, 100, 1000, 5000): bench(git_bits, r, key, 0, num=MAX, times=Times) bench(get_bits_pipe, pipe, key, 0, num=MAX, times=Times) for i in range(MAX): r.setbit(key, i, 1) print("all true, times %d" % Times) for MAX in (1, 10, 100, 1000, 5000): bench(git_bits, r, key, 0, num=MAX, times=Times) bench(get_bits_pipe, pipe, key, 0, num=MAX, times=Times)