import codecs import math import operator import re from collections import Counter, defaultdict from tkinter import _flatten import jieba import numpy as np import pandas as pd from sklearn.feature_extraction.text import (CountVectorizer, TfidfTransformer, TfidfVectorizer) from sklearn.linear_model import LogisticRegression from sklearn.metrics import (accuracy_score, f1_score, precision_score, recall_score) from sklearn.naive_bayes import BernoulliNB """ @author: yaoyue123 2019/11/13 社会计算（跨）第一次作业demo """ min_freq = 5 def read_train_valid(filename): """ 读取训练或者验证文件 :param filename: 训练集/验证集的文件名字 :return: 返回训练集的文本和标签 其中文本是一个list, 标签是一个list(每个元素为int) 返回示例：['我很开心', '你不是真正的快乐', '一切都是假的], [1, 0, 0] """ fp = pd.read_table(filename, sep='\t', error_bad_lines=False) return fp[fp.columns[1]].tolist(), fp[fp.columns[0]].tolist() def read_test(filename): """ 读取测试文件 :param filname: 测试集文件名字 :return: 返回测试集文本和对应的id编号 其中文本是一个list, id就是文件中的id 返回示例：['我很开心', '你不是真正的快乐', '一切都是假的], [1,2,3] """ fp = pd.read_table(filename, sep='\t', error_bad_lines=False) return fp[fp.columns[1]].tolist(), fp[fp.columns[0]].tolist() def split_text(text_data): """ 将原始文本分词 例如：输入['今天星期四', '明天有雨'], 返回[['今天','星期四'], ['明天','有','雨']] 或者例如：输入['I don't want to go', ''Who are you?’], 返回[['I', 'don't', 'want', 'to', 'go',],['Who', ' are', 'you',' ?']] 中文推荐jieba分词(请pip install jieba)，英文直接用空格切分句子即可 :param text_data: :return: """ sentences_split = [] for index in range(len(text_data)): word = list(jieba.cut(text_data[index])) sentences_split.append(word) # 加载停用词表 stopword = open(r'stop.txt', 'r', encoding='utf-8').read().split() # 去除停用词 for words in sentences_split: for word in words: if word in stopword: words.remove(word) return sentences_split def tfidf(words_vec): x_words = [] for words in words_vec: x_words.append(" ".join(words)) vectorizer = CountVectorizer(max_features=10) # 该类会统计每个词语的tf-idf权值 tf_idf_transformer = TfidfTransformer() # 将文本转为词频矩阵并计算tf-idf tf_idf = tf_idf_transformer.fit_transform( vectorizer.fit_transform(x_words)) # 将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重 x_words_weight = tf_idf.toarray() return x_words_weight def vectorizer(train_data, valid_data, test_data): """ 将原始文本转化为向量 可选方法 1. one hot 方法, 2. tf-idf方法，3. 词向量或者其他方法 推荐使用1或者2的方法，两种方法都可以选择调用第三方包或者自己实现 可用的包： CountVectorizer, TfidfVectorizer可以实现one-hot和tfd-idf编码 使用tf-idf比onehot有加分; 自己实现比调用第三方包有额外加分 愿意使用词向量也可 返回示例：[[0,0,1,2], [2,3,3,4]] [[0,0,1,2], [2,3,3,4]] [[0,0,1,2], [2,3,3,4]] 分别对应train, valid, test的向量化结果，每一个结果都是一个二维的list, 每一个元素都是int or float :param text: 一个list, 每个元素是一句话 :return: 3个二维list """ """ train_dict = feature_select(train_data) train_list = [] for docs in train_data: list_new = [] for item in docs: list_new.append(train_dict[item]) train_list.append(list_new) # print(train_dict[item]) print(train_list) """ return tfidf(train_data), tfidf(valid_data), tfidf(test_data) def train_valid(train_x, train_label, valid_x, valid_label): """ 开始训练模型并使用验证集验证效果 推荐方法 1. 朴素贝叶斯: 可用包 BernoulliNB, 2. LogisticRegression 可用包 sklearn中的LogisticsRegression 3. 其他方法不做强制性要求 以上方法，自己实现比调包加分，手动实现 LR比手动实现朴素贝叶斯加分 调包示例： model = one_model() model.some_func(train_data, train_label) score = model.score_function(valid_data, valid_label) print("valid accuracy score is {:.4f}".format(score} 以上只计算了accuracy score，如果能打印验证集precision_score, recall_score, F1_score， 加分 可以调用 sklearn.metric 中的precision_score, recall_score, F1_score 请试着调节模型的参数，将你的各项score尽量提高，验证集分数越高，测试集的效果也可能越好 也推荐同学们尝试交叉验证方法，提升模型的泛化能力 最终返回训练完成的模型 model :param train_data: :param train_label: :param valid_data: :param valid_label: :return: """ # 定义模型 model = LogisticRegression() # 开始训练,使用训练集的train_x 和train_y model.fit(train_x, train_label) # 训练完毕，预测一下验证集 prediction = model.predict(valid_x) # 打印预测的结果和得分 print(prediction) print('Accuracy score: {:.4f}'.format( accuracy_score(valid_label, prediction))) print('Precision score: {:.4f}'.format( precision_score(valid_label, prediction, average='macro'))) print('Recall score: {:.4f}'.format( recall_score(valid_label, prediction, average='macro'))) print('F1 score: {:.4f}'.format( f1_score(valid_label, prediction, average='macro'))) return model def predict(mode, test_data): """ 使用训练好的模型预测测试数据 返回预测的标签，为list :param mode: :param test_data: :return: """ pass def run_step(): """ 选择相应的任务和文件 读文件，train_data, train_label = some_function(filename='') valid_data, validlabel = some_function(filename='') test_data, test_ids = some_function(filename='') 将原始文本分词： train_data = split_function(train_data) valid_data = split_function(valid_data) test_data = split_function(test_data) 将分词后的文本变成向量： train_vec, valid_vec, test_vec = vectorizer(train_data, valid_data, test_data) 开始训练模型： model = train_function(train_vec, train_label, valid_vec, valid_label) 开始预测结果： test_result = predict(mode, test_data) 写入文件： 将test和对应的test写入test_result, 结果样本如submit所示 请大家尽量按照步骤来，顺利完成整个流程。有余力的同学可以试试提高预测的精度 :return: """ train_sentences, train_label = read_train_valid(filename=r'training.txt') test_sentences, test_id = read_test(filename=r'test.txt') valid_sentences, valid_label = read_train_valid(filename=r'validation.txt') train_sentences_split = split_text(train_sentences) test_sentences_split = split_text(test_sentences) valid_sentences_split = split_text(valid_sentences) # 遍历所有的句子，并打印编号和句子内容 #print(1, train_sentences, train_label) #print(2, test_sentences, test_id) #print(3, valid_sentences, valid_label) #print(4, train_sentences_split) # print(5, vectorizer(train_sentences_split, # valid_sentences_split, test_sentences_split)) train_x, valid_x, test_x = vectorizer(train_sentences_split, valid_sentences_split, test_sentences_split) model = train_valid(train_x, train_label, valid_x, valid_label) result = model.predict(test_x) # 打印预测结果 print("test result", result) # 将结果和id写入文件夹中 with open("submit.txt", "w", encoding='utf-8') as f: f.write("id\tlabels\n") for i in range(len(result)): idx = str(test_id[i]) lable = str(result[i]) f.write(idx + '\t' + lable + '\n') if __name__ == '__main__': run_step()