TF-IDF（词频--逆文档）算法的实现_tfdif

TF-IDF算法实例

import numpy as np
import pandas as pd
import math
1
2
3

1.定义数据和预处理

docA = "The cat sat on my bed"
docB = "This dog sat in the trees"

bowA = docA.split(" ")
bowB= docB.split(" ")

#构建词库
wordSet = set(bowA).union(set(bowB))
wordSet  #去重后的单词有哪些
1
2
3
4
5
6
7
8
9

2.进行词数统计哦

# 用统计字典来保存词出现的次数
wordDictA = dict.fromkeys(wordSet,0)
wordDictB = dict.fromkeys(wordSet,0)
#print(wordDictA)
'''
显示结果：
{'The': 0, 'dog': 0, 'trees': 0, 'cat': 0, 'bed': 0,'on': 0, 'my': 0, 'in': 0, 'the': 0, 'sat': 0, 'This': 0}

'''
#wordDictA
print('---------------------------------------------------\n')
#wordDictB

# 遍历文档，对各个输入词进行统计
for word in bowA:
    wordDictA[word] += 1

for word in bowB:
    wordDictB[word] += 1
    
#wordDictA
print('---------------------------------------------------\n')
#wordDictB
'''
显示结果：
{'The': 1,
 'dog': 0,
 'trees': 0,
 'cat': 1,
 'bed': 1,
 'on': 1,
 'my': 1,
 'in': 0,
 'the': 0,
 'sat': 1,
 'This': 0}
 
 
'''
pd.DataFrame([wordDictA,wordDictB])
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

3.计算-----词频TF

def computeTF(wordDict,bow):
    #用一个字典对象记录tf
    tfDict = {}
    nbowCount = len(bow)
    
    for word,count in wordDict.items():
        tfDict[word ] = count / nbowCount
    return tfDict
1
2
3
4
5
6
7
8

进行一下测试

tfA = computeTF(wordDictA,bowA)
tfB = computeTF(wordDictB,bowB)
print('---------------------------------------------------\n')# 在这条语句前面，好像就不会被执行起来，很奇怪啊，TNND
tfA

1
2
3
4
5

4.计算—逆文档频率IDF

def computeIDF(wordDictList):
    #用一个字典对象保存IDF结果，每个词都作为key,初始值为0
    idfDict = dict.fromkeys(wordDictList[0],0)
    N = len(wordDictList)
    
    for worDict in wordDictList:
        #遍历字典中的每个词汇
        for word,count in worDict.items():
            if count>0:
                #先把Ni增加1，存入到idfDict里面
                idfDict[word]+=1
                
    #已经得到所有词汇i对应的Ni，现在根据公式把它替换成为idf的值
    for word,Ni in  idfDict.items():
        idfDict[word] = math.log10((N+1)/(Ni+1))
        
    return idfDict
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

进行一下测试

idfs = computeIDF([wordDictA,wordDictB])
idfs
1
2

5.计算TF-IDF

def computeTF_IDF(tf,idfs):
    tfidf = {}
    for word ,tfval in tf.items():
        tfidf[word] = tfval*idfs[word]
    return tfidf
1
2
3
4
5

最后测试

tfdifA = computeTF_IDF(tfA,idfs)
tfidfB = computeTF_IDF(tfB,idfs)

pd.DataFrame([tfdifA,tfidfB])
1
2
3
4