20Congress_lec

import jieba
import numpy as np
#安装wordcloud时可能会有依存冲突
from wordcloud import WordCloud   #这个包需要matplotlib
from collections import Counter  #单词计数
import matplotlib.pyplot as plt  #画图
import imageio  #图片输入输出

#读文件
data={}
text_file = open('./data/20Congress.txt','r',encoding='utf-8')
text = text_file.read()
with open('./data/stopwords.txt',encoding='utf-8') as file:
    stopwords = {line.strip() for line in file}

#统计词频
seg_list = jieba.cut(text, cut_all=False)
for word in seg_list:
    if len(word)>=2:    #只取词长大于2的词，过滤掉“，”，“。”等
        if not data.__contains__(word):
            data[word]=0
        data[word]+=1
print(data) 

import imageio
my_wordcloud = WordCloud(  
    background_color='white',  #设置背景颜色
    max_words=400,  #设置最大实现的字数
    font_path=r'./data/SimHei.ttf',  #设置字体格式，如不设置显示不了中文
    mask=imageio.imread('./data/mapofChina.jpg'),
    width=1000,   #像素
    height=1000,
    stopwords = stopwords
).generate_from_frequencies(data)    #根据频率生成
plt.figure(figsize=(5,5))     #输出图片大小
plt.imshow(my_wordcloud)
plt.axis('off')   #不要坐标轴
my_wordcloud.to_file('result.jpg')
text_file.close()

Chinese text procesdsing_lec

分词工具

import jieba
seg_list = jieba.cut("华中农业大学坐落在武汉南湖狮子山上", cut_all=True)
print("全模式: " + "/ ".join(seg_list))  # 全模式

seg_list = jieba.cut("华中农业大学坐落在武汉南湖狮子山上", cut_all=False)
print("精确模式: " + "/ ".join(seg_list))  # 精确模式

seg_list = jieba.cut("华中农业大学坐落在武汉南湖狮子山上")  # 默认是精确模式
print(", ".join(seg_list))

添加自定义词典

text ="李一一告诉我，华中农业大学校训是勤读立耕立己达人"
# 全模式
seg_list = jieba.cut(text, cut_all=True)
print(u"[全模式]: ", "/ ".join(seg_list))

# 精确模式
seg_list = jieba.cut(text, cut_all=False)
print(u"[精确模式]: ", "/ ".join(seg_list))

jieba.load_userdict("./data/mydict.txt") #需UTF-8，可以在另存为里面设置

#也可以用jieba.add_word(" ")添加词典

text ="李一一告诉我，华中农业大学校训是勤读立耕立己达人"
# 全模式
seg_list = jieba.cut(text, cut_all=True)
print(u"[全模式]: ", "/ ".join(seg_list))

# 精确模式
seg_list = jieba.cut(text, cut_all=False)
print(u"[精确模式]: ", "/ ".join(seg_list))

关键词抽取

import jieba.analyse

seg_list = jieba.cut(text, cut_all=False)
print (u"分词结果:")
print ("/".join(seg_list))

#获取关键词
tags = jieba.analyse.extract_tags(text, topK=5)
print (u"关键词:")
print (" ".join(tags))

#from jieba import analyse
#import jieba.analyse
tags = jieba.analyse.extract_tags(text, topK=5, withWeight=True)
for word, weight in tags:
    print(word, weight)

词性标注

import jieba.posseg as pseg
words = pseg.cut("华中农业大学坐落在武汉南湖狮子山上")
for word, flag in words:
    print("%s %s" % (word, flag))

词云展示

import jieba
from wordcloud import WordCloud   #需要安装
#from scipy.misc import imread
from collections import Counter
import matplotlib.pyplot as plt


import imageio
#img = imageio.imread('data/duck.jpg')

data={}
text_file = open('./data/19Congress.txt','r',encoding='utf-8')
text = text_file.read()
with open('./data/stopwords.txt',encoding='utf-8') as file:
    stopwords = {line.strip() for line in file}
seg_list = jieba.cut(text, cut_all=False)
for word in seg_list:
    if len(word)>=2:
        if not data.__contains__(word):
            data[word]=0
        data[word]+=1
print(data)   

pip install imageio

import imageio
my_wordcloud = WordCloud(  
    background_color='white',  #设置背景颜色
    max_words=400,  #设置最大实现的字数
    font_path=r'./data/SimHei.ttf',  #设置字体格式，如不设置显示不了中文
    #mask=imread('./data/mapofChina.jpg'), #指定在什么图片上画
    mask=imageio.imread('./data/mapofChina.jpg'),
    width=1000,   #像素
    height=1000,
    stopwords = stopwords
).generate_from_frequencies(data)    #根据频率生成

plt.figure(figsize=(10,10))     #输出图片大小
plt.imshow(my_wordcloud)
plt.axis('off')   #不要坐标轴
my_wordcloud.to_file('result.jpg')
text_file.close()

English text processing_lec

NLTK工具包安装

import nltk #pip install nltk
#nltk.download()  #可通过该命令下载，放到默认路径下，如anaconda3\envs\nlp\nltk_data
#https://github.com/nltk/nltk_data/tree/gh-pages  #也可以从github找到指定资源

分词

import nltk
from nltk.tokenize import word_tokenize    #分词包
from nltk.text import Text

input_str = "Today's weather is good, very windy and sunny, we have no classes in the afternoon,We have to play basketball tomorrow."

import nltk
tokens = word_tokenize(input_str)

tokens = [word.lower() for word in tokens]

import nltk
help(nltk.text)

t = Text(tokens)    #创建一个Text对象，方便后续操作

t.count('good')

t.index('good')

t.plot(5)    #统计token中每个词的词频

停用词

from nltk.corpus import stopwords
nltk.download("stopwords")
stopwords.readme().replace('\n', ' ')

stopwords.fileids()  #各个语系的停用词

stopwords.raw('english').replace('\n',' ')

test_words = [word.lower() for word in tokens]
test_words_set = set(test_words)

test_words_set.intersection(set(stopwords.words('english')))

filtered = [w for w in test_words_set if(w not in stopwords.words('english'))]

词性标注

from nltk import pos_tag
tags = pos_tag(tokens)

分块

from nltk.chunk import RegexpParser

sentence = [('the','DT'),('little','JJ'),('yellow','JJ'),('dog','NN'),('died','VBD')]
grammer = "NP: {<DT>?<JJ>*<NN>}"
cp = nltk.RegexpParser(grammer) #生成规则
result = cp.parse(sentence) #进行分块
print(result)

result.draw() #调用matplotlib库画出来

命名实体识别

nltk.download() 
from nltk import ne_chunk
sentence = "Edison went to Hongkong today."
print(ne_chunk(pos_tag(word_tokenize(sentence))))

词干提取

from nltk.stem.porter import PorterStemmer
porter_stemmer = PorterStemmer()

porter_stemmer.stem('activation')

porter_stemmer.stem('activated')

porter_stemmer.stem('activated')

#安装WordNet
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
wordnet_lemmatizer.lemmatize('dogs')

wordnet_lemmatizer.lemmatize('activating')

数据清洗

import re
from nltk.corpus import stopwords
# 输入数据
s = '    RT @Amila #Test\nTom\'s newly listed Co  & Mary\'s unlisted     Group to supply tech for nlTK.\nh $TSLA $AAPL https:// t.co/x34afsfQsh'

#指定停用词
cache_english_stopwords = stopwords.words('english')

def text_clean(text):
    print('原始数据:', text, '\n')
    
    # 去掉HTML标签(e.g. &)
    text_no_special_entities = re.sub(r'\&\w*;|#\w*|@\w*', '', text)
    print('去掉特殊标签后的:', text_no_special_entities, '\n')
    
    # 去掉一些价值符号
    text_no_tickers = re.sub(r'\$\w*', '', text_no_special_entities) 
    print('去掉价值符号后的:', text_no_tickers, '\n')
    
    # 去掉超链接
    text_no_hyperlinks = re.sub(r'https?:\/\/.*\/\w*', '', text_no_tickers)
    print('去掉超链接后的:', text_no_hyperlinks, '\n')

    # 去掉一些专门名词缩写，简单来说就是字母比较少的词
    text_no_small_words = re.sub(r'\b\w{1,2}\b', '', text_no_hyperlinks) 
    print('去掉专门名词缩写后:', text_no_small_words, '\n')
    
    # 去掉多余的空格
    text_no_whitespace = re.sub(r'\s\s+', ' ', text_no_small_words)
    text_no_whitespace = text_no_whitespace.lstrip(' ') 
    print('去掉空格后的:', text_no_whitespace, '\n')
    
    # 分词
    tokens = word_tokenize(text_no_whitespace)
    print('分词结果:', tokens, '\n')    
          
    # 去停用词
    list_no_stopwords = [i for i in tokens if i not in cache_english_stopwords]
    print('去停用词后结果:', list_no_stopwords, '\n')
    
    # 过滤后结果
    text_filtered =' '.join(list_no_stopwords) # ''.join() would join without spaces between words.
    print('过滤后:', text_filtered)

text_clean(s)