仅用不到 150 行代码，我开发出了一个搜索引擎

<doc>    <title>Wikipedia: London Beer Flood</title>    <url>https://en.wikipedia.org/wiki/London_Beer_Flood</url>    <abstract>The London Beer Flood was an accident at Meux & Co's Horse Shoe Brewery, London, on 17 October 1814. It took place when one of the  wooden vats of fermenting porter burst.</abstract>    ...</doc>

from dataclasses import dataclass@dataclassclass Abstract:    """Wikipedia abstract"""    ID: int    title: str    abstract: str    url: str    @property    def fulltext(self):        return ' '.join([self.title, self.abstract])

import gzipfrom lxml import etreefrom search.documents import Abstractdef load_documents():    # open a filehandle to the gzipped Wikipedia dump    with gzip.open('data/enwiki.latest-abstract.xml.gz', 'rb') as f:        doc_id = 1        # iterparse will yield the entire `doc` element once it finds the        # closing `</doc>` tag        for _, element in etree.iterparse(f, events=('end',), tag='doc'):            title = element.findtext('./title')            url = element.findtext('./url')            abstract = element.findtext('./abstract')            yield Abstract(ID=doc_id, title=title, url=url, abstract=abstract)            doc_id += 1            # the `element.clear()` call will explicitly free up the memory            # used to store the element            element.clear()

{    ...    "london": [5245250, 2623812, 133455, 3672401, ...],    "beer": [1921376, 4411744, 684389, 2019685, ...],    "flood": [3772355, 2895814, 3461065, 5132238, ...],    ...}

import StemmerSTEMMER = Stemmer.Stemmer('english')def tokenize(text):    return text.split()def lowercase_filter(text):    return [token.lower() for token in tokens]def stem_filter(tokens):    return STEMMER.stemWords(tokens)

import reimport stringPUNCTUATION = re.compile('[%s]' % re.escape(string.punctuation))def punctuation_filter(tokens):    return [PUNCTUATION.sub('', token) for token in tokens]

# top 25 most common words in English and "wikipedia":# https://en.wikipedia.org/wiki/Most_common_words_in_EnglishSTOPWORDS = set(['the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',                 'I', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',                 'do', 'at', 'this', 'but', 'his', 'by', 'from', 'wikipedia'])def stopword_filter(tokens):    return [token for token in tokens if token not in STOPWORDS]

def analyze(text):    tokens = tokenize(text)    tokens = lowercase_filter(tokens)    tokens = punctuation_filter(tokens)    tokens = stopword_filter(tokens)    tokens = stem_filter(tokens)    return [token for token in tokens if token]

class Index:    def __init__(self):        self.index = {}        self.documents = {}    def index_document(self, document):        if document.ID not in self.documents:            self.documents[document.ID] = document        for token in analyze(document.fulltext):            if token not in self.index:                self.index[token] = set()            self.index[token].add(document.ID)

def _results(self, analyzed_query):    return [self.index.get(token, set()) for token in analyzed_query]def search(self, query):    """    Boolean search; this will return documents that contain all words from the    query, but not rank them (sets are fast, but unordered).    """    analyzed_query = analyze(query)    results = self._results(analyzed_query)    documents = [self.documents[doc_id] for doc_id in set.intersection(*results)]    return documentsIn [1]: index.search('London Beer Flood')search took 0.16307830810546875 millisecondsOut[1]:[Abstract(ID=1501027, title='Wikipedia: Horse Shoe Brewery', abstract='The Horse Shoe Brewery was an English brewery in the City of Westminster that was established in 1764 and became a major producer of porter, from 1809 as Henry Meux & Co. It was the site of the London Beer Flood in 1814, which killed eight people after a porter vat burst.', url='https://en.wikipedia.org/wiki/Horse_Shoe_Brewery'), Abstract(ID=1828015, title='Wikipedia: London Beer Flood', abstract="The London Beer Flood was an accident at Meux & Co's Horse Shoe Brewery, London, on 17 October 1814. It took place when one of the  wooden vats of fermenting porter burst.", url='https://en.wikipedia.org/wiki/London_Beer_Flood')]

def search(self, query, search_type='AND'):    """    Still boolean search; this will return documents that contain either all words    from the query or just one of them, depending on the search_type specified.    We are still not ranking the results (sets are fast, but unordered).    """    if search_type not in ('AND', 'OR'):        return []    analyzed_query = analyze(query)    results = self._results(analyzed_query)    if search_type == 'AND':        # all tokens must be in the document        documents = [self.documents[doc_id] for doc_id in set.intersection(*results)]    if search_type == 'OR':        # only one token has to be in the document        documents = [self.documents[doc_id] for doc_id in set.union(*results)]    return documentsIn [2]: index.search('London Beer Flood', search_type='OR')search took 0.02816295623779297 secondsOut[2]:[Abstract(ID=5505026, title='Wikipedia: Addie Pryor', abstract='| birth_place    = London, England', url='https://en.wikipedia.org/wiki/Addie_Pryor'), Abstract(ID=1572868, title='Wikipedia: Tim Steward', abstract='|birth_place         = London, United Kingdom', url='https://en.wikipedia.org/wiki/Tim_Steward'), Abstract(ID=5111814, title='Wikipedia: 1877 Birthday Honours', abstract='The 1877 Birthday Honours were appointments by Queen Victoria to various orders and honours to reward and highlight good works by citizens of the British Empire. The appointments were made to celebrate the official birthday of the Queen, and were published in The London Gazette on 30 May and 2 June 1877.', url='https://en.wikipedia.org/wiki/1877_Birthday_Honours'), ...In [3]: len(index.search('London Beer Flood', search_type='OR'))search took 0.029065370559692383 secondsOut[3]: 49627

# in documents.pyfrom collections import Counterfrom .analysis import analyze@dataclassclass Abstract:    # snip    def analyze(self):        # Counter will create a dictionary counting the unique values in an array:        # {'london': 12, 'beer': 3, ...}        self.term_frequencies = Counter(analyze(self.fulltext))    def term_frequency(self, term):        return self.term_frequencies.get(term, 0)

# in index.py we add `document.analyze()def index_document(self, document):    if document.ID not in self.documents:        self.documents[document.ID] = document        document.analyze()

def search(self, query, search_type='AND', rank=True):    # snip    if rank:        return self.rank(analyzed_query, documents)    return documentsdef rank(self, analyzed_query, documents):    results = []    if not documents:        return results    for document in documents:        score = sum([document.term_frequency(token) for token in analyzed_query])        results.append((document, score))    return sorted(results, key=lambda doc: doc[1], reverse=True)

# index.pyimport mathdef document_frequency(self, token):    return len(self.index.get(token, set()))def inverse_document_frequency(self, token):    # Manning, Hinrich and Schütze use log10, so we do too, even though it    # doesn't really matter which log we use anyway    # https://nlp.stanford.edu/IR-book/html/htmledition/inverse-document-frequency-1.html    return math.log10(len(self.documents) / self.document_frequency(token))def rank(self, analyzed_query, documents):    results = []    if not documents:        return results    for document in documents:        score = 0.0        for token in analyzed_query:            tf = document.term_frequency(token)            idf = self.inverse_document_frequency(token)            score += tf * idf        results.append((document, score))    return sorted(results, key=lambda doc: doc[1], reverse=True)