Python 编写知乎爬虫实践

class BloomFilter :

def __init__ ( self ):

# Initialize bloom filter, set size and all bits to 0

bit_array = bitarray ( BIT_SIZE )

bit_array . setall ( 0 )

self . bit_array = bit_array

def add ( self , url ):

# Add a url, and set points in bitarray to 1 (Points count is equal to hash funcs count.)

# Here use 7 hash functions.

point_list = self . get_postions ( url )

for b in point_list :

self . bit_array [ b ] = 1

def contains ( self , url ):

# Check if a url is in a collection

point_list = self . get_postions ( url )

result = True

for b in point_list :

result = result and self . bit_array [ b ]

return result

def get_postions ( self , url ):

# Get points positions in bit vector.

point1 = mmh3 . hash ( url , 41 ) % BIT_SIZE

point2 = mmh3 . hash ( url , 42 ) % BIT_SIZE

point3 = mmh3 . hash ( url , 43 ) % BIT_SIZE

point4 = mmh3 . hash ( url , 44 ) %