'C:\\Users\\user\\PycharmProjects\\untitled\Тест 2.txt',
'C:\\Users\\user\\PycharmProjects\\untitled\Тест 3.txt',
'C:\\Users\\user\\PycharmProjects\\untitled\Тест 4.txt',
'C:\\Users\\user\\PycharmProjects\\untitled\Тест 5.txt']
tests=[]
for file in test_files:
tests.append([])
f=open(file,'r',encoding='utf-8').read()
for k in sent_tokenize(f):
file_ru=[]
file_ru.append(k)
if file_ru!=[]:
tests[-1].append(lemmatise(file_ru))
print (tests)
df_pos = pd.read_csv('positive.csv', sep=';', header=None, names=["id","tdate","tname","ttext","ttype","trep","trtw","tfav","tstcount","tfoll","tfrien","listcount"])
df_pos['class'] = 1
df_neg = pd.read_csv('negative.csv', sep=';', header=None, names=["id","tdate","tname","ttext","ttype","trep","trtw","tfav","tstcount","tfoll","tfrien","listcount"])
df_neg['class'] = -1
res_df = pd.concat([df_pos, df_neg])
twit_X1 = res_df['ttext'].values
twit_y = res_df['class'].values
print (len(twit_X1))
twit_X=lemmatise(twit_X1)
twit_X=np.array(twit_X)
kf = KFold(n_splits=3,shuffle=True)
for train_index, test_index in kf.split(twit_X):
X_train, X_test = twit_X[train_index], twit_X[test_index]
y_train, y_test = twit_y[train_index], twit_y[test_index]
B_vectorizer = TfidfVectorizer(max_df=0.5, min_df=2, use_idf=True)
X_transformed_train = B_vectorizer.fit_transform(X_train)
Bayes_clf = MultinomialNB()
Bayes_clf.fit(X_transformed_train, y_train) # Обучаем его
y_test_predict = Bayes_clf.predict(B_vectorizer.transform(X_test))
y_predicted = Bayes_clf.predict(B_vectorizer.transform(X_test))
print (X_test[0],Bayes_clf.predict_proba(B_vectorizer.transform(X_test))[0],y_test_predict[0])
print("ACcuracy: {0}".format(accuracy_score(y_test, y_test_predict)))
print("Precision: {0}".format(precision_score(y_test, y_predicted)))
print("Recall: {0}".format(recall_score(y_test, y_predicted)))
print("F1-measure: {0}".format(f1_score(y_test, y_predicted)))
import pickle
bayes=pickle.dumps(Bayes_clf)
for testf in tests:
print ('test numb',tests.index(testf)+1)
for sent in testf:
print (test(sent,Bayes_clf,B_vectorizer))
kf = KFold(n_splits=2,shuffle=True)
for train_index, test_index in kf.split(twit_X):
X_train, X_test = twit_X[train_index], twit_X[test_index]
y_train, y_test = twit_y[train_index], twit_y[test_index]
rf_vectorizer = TfidfVectorizer(max_df=0.5, min_df=2, use_idf=True)
X_transformed_train = rf_vectorizer.fit_transform(X_train)
rf_clf= RandomForestClassifier(n_estimators = 100)
rf_clf.fit(X_transformed_train, y_train)
y_predicted = rf_clf.predict(rf_vectorizer.transform(X_test))
print("Accuracy: {0}".format(accuracy_score(y_test, y_predicted)))
print("Precision: {0}".format(precision_score(y_test, y_predicted)))
print("Recall: {0}".format(recall_score(y_test, y_predicted)))
print("F1-measure: {0}".format(f1_score(y_test, y_predicted)))
print()
import pickle
forest=pickle.dumps(rf_clf)
for testf in tests:
print ('test numb',tests.index(testf)+1)
for sent in testf:
print (test(sent,rf_clf,rf_vectorizer))
for train_index, test_index in kf.split(twit_X):
X_train, X_test = twit_X[train_index], twit_X[test_index]
y_train, y_test = twit_y[train_index], twit_y[test_index]
lg_vectorizer = TfidfVectorizer(max_df=0.5, min_df=2, use_idf=True)
X_transformed_train = lg_vectorizer.fit_transform(X_train)
lg_clf= LogisticRegression()
lg_clf.fit(X_transformed_train, y_train)
y_predicted = lg_clf.predict(lg_vectorizer.transform(X_test))
print("Accuracy: {0}".format(accuracy_score(y_test, y_predicted)))
print("Precision: {0}".format(precision_score(y_test, y_predicted)))
print("Recall: {0}".format(recall_score(y_test, y_predicted)))
print("F1-measure: {0}".format(f1_score(y_test, y_predicted)))
print()
regr=pickle.dumps(lg_clf)
for testf in tests:
print ('test numb',tests.index(testf)+1)
for sent in testf:
print (test(sent,lg_clf,lg_vectorizer))