|
@@ -0,0 +1,257 @@
|
|
1
|
+from sklearn import datasets
|
|
2
|
+from sklearn.model_selection import train_test_split, KFold
|
|
3
|
+from sklearn.neural_network import MLPClassifier
|
|
4
|
+from sklearn.svm import SVC
|
|
5
|
+from sklearn.neighbors import KNeighborsClassifier
|
|
6
|
+from sklearn.metrics import precision_score, recall_score, plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay
|
|
7
|
+import numpy as np
|
|
8
|
+import matplotlib.pyplot as plt
|
|
9
|
+from random import seed, randint
|
|
10
|
+from graphs import bars_plot2, curve_plot
|
|
11
|
+from statistics import mean
|
|
12
|
+import time
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+np.random.seed(0) # initialise le generateur aletoire avec seed=0
|
|
16
|
+seed(0)
|
|
17
|
+
|
|
18
|
+#######################
|
|
19
|
+### TP3 PROGRAMME ###
|
|
20
|
+#######################
|
|
21
|
+# Chargement de la base de données MNIST
|
|
22
|
+mnist = datasets.fetch_openml('mnist_784')
|
|
23
|
+
|
|
24
|
+# Division de la base de données
|
|
25
|
+x_train = mnist.data[:3500,:] ## training
|
|
26
|
+y_train = mnist.target[:3500]
|
|
27
|
+x_test = mnist.data[3500:5001,:] ## test
|
|
28
|
+y_test = mnist.target[3500:5001]
|
|
29
|
+
|
|
30
|
+# print(x_train.shape,x_test.shape) #
|
|
31
|
+
|
|
32
|
+# Mise en conformité des dimensions de xtrain et xtest aux dimensions attendues par le classifeur
|
|
33
|
+x_train = x_train.reshape((-1, 784))
|
|
34
|
+x_test = x_test.reshape((-1, 784))
|
|
35
|
+
|
|
36
|
+def kernel_variation():
|
|
37
|
+ kernels = ["linear","poly","rbf","sigmoid"]
|
|
38
|
+ scores = {}
|
|
39
|
+ for kernel in kernels:
|
|
40
|
+ clsvm = SVC(kernel=kernel)
|
|
41
|
+ clsvm.fit(X=x_train, y=y_train)
|
|
42
|
+ scores[kernel] = clsvm.score(X=x_test, y=y_test) * 100
|
|
43
|
+ print("Score avec kernel=%s: %.2f"%(kernel, scores[kernel]))
|
|
44
|
+ bars_plot2(bar_labels=kernels, bar_heights=list(scores.values()), xlabel="Kernels utilisés",
|
|
45
|
+ ylabel="Score", fig_title="Score par kernel utilisé", show=True, percent_mark=True,
|
|
46
|
+ output="tp3Output/scores_for_kernel_variation.png")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+def kernel_and_c_variation():
|
|
50
|
+ kernels = ["linear", "poly", "rbf", "sigmoid"]
|
|
51
|
+ scores = {}
|
|
52
|
+ c_values = [0.1 + 0.18 * x for x in range(6)]
|
|
53
|
+ for kernel in kernels:
|
|
54
|
+ scores[kernel] = {}
|
|
55
|
+ for c_value in c_values:
|
|
56
|
+ clsvm = SVC(kernel=kernel, C=c_value)
|
|
57
|
+ clsvm.fit(X=x_train, y=y_train)
|
|
58
|
+ scores[kernel][c_value] = clsvm.score(X=x_test, y=y_test) * 100
|
|
59
|
+ print("Score avec kernel=%s: %.2f" % (kernel, clsvm.score(X=x_test, y=y_test) * 100))
|
|
60
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
61
|
+ plt.plot(c_values, list(scores["linear"].values()), label="linear")
|
|
62
|
+ plt.plot(c_values, list(scores["poly"].values()), label="poly")
|
|
63
|
+ plt.plot(c_values, list(scores["rbf"].values()), label="rbf")
|
|
64
|
+ plt.plot(c_values, list(scores["sigmoid"].values()), label="sigmoid")
|
|
65
|
+ plt.xlabel("Valeurs de C")
|
|
66
|
+ plt.xlim(left=0.1)
|
|
67
|
+ plt.ylabel("Score (précision)")
|
|
68
|
+ plt.title("Courbe d'évolution de la précision en fonction de C")
|
|
69
|
+ plt.legend()
|
|
70
|
+ plt.savefig("tp3Output/scores_all_kernel_with_c_variation.png", dpi="figure")
|
|
71
|
+ plt.show()
|
|
72
|
+ plt.close()
|
|
73
|
+
|
|
74
|
+def train_test_error_with_c_variation():
|
|
75
|
+ kernels = ["linear", "poly", "rbf", "sigmoid"]
|
|
76
|
+ scores = {}
|
|
77
|
+ c_values = [0.1 + 0.18 * x for x in range(6)]
|
|
78
|
+ data_types = ["train", "test"]
|
|
79
|
+ for data_type in data_types:
|
|
80
|
+ scores[data_type] = {}
|
|
81
|
+ for kernel in kernels:
|
|
82
|
+ scores[data_type][kernel] = {}
|
|
83
|
+ for c_value in c_values:
|
|
84
|
+ clsvm = SVC(kernel=kernel, C=c_value)
|
|
85
|
+ clsvm.fit(X=x_train, y=y_train)
|
|
86
|
+ if data_type=="train":
|
|
87
|
+ scores[data_type][kernel][c_value] = (1 - clsvm.score(X=x_train, y=y_train)) * 100
|
|
88
|
+ else:
|
|
89
|
+ scores[data_type][kernel][c_value] = (1 - clsvm.score(X=x_test, y=y_test)) * 100
|
|
90
|
+ print("Score d'erreur données=%s avec kernel=%s: %.2f" % (data_type, kernel, scores[data_type][kernel][c_value]))
|
|
91
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
92
|
+ plt.plot(c_values, list(scores["train"]["linear"].values()), label="train linear", marker="o")
|
|
93
|
+ plt.plot(c_values, list(scores["train"]["poly"].values()), label="train poly", marker="o")
|
|
94
|
+ plt.plot(c_values, list(scores["train"]["rbf"].values()), label="train rbf", marker="o")
|
|
95
|
+ plt.plot(c_values, list(scores["train"]["sigmoid"].values()), label="train sigmoid", marker="o")
|
|
96
|
+ plt.plot(c_values, list(scores["test"]["linear"].values()), label="test linear", marker="*")
|
|
97
|
+ plt.plot(c_values, list(scores["test"]["poly"].values()), label="test poly", marker="*")
|
|
98
|
+ plt.plot(c_values, list(scores["test"]["rbf"].values()), label="test rbf", marker="*")
|
|
99
|
+ plt.plot(c_values, list(scores["test"]["sigmoid"].values()), label="test sigmoid", marker="*")
|
|
100
|
+ plt.xlabel("Valeurs de C")
|
|
101
|
+ plt.xlim(left=0.1)
|
|
102
|
+ plt.ylabel("Error score")
|
|
103
|
+ plt.title("Courbe d'évolution du pourcentage d'erreur de précision en fonction de C")
|
|
104
|
+ plt.legend()
|
|
105
|
+ plt.savefig("tp3Output/error_scores_train_test_with_kernel_and_c_variation.png", dpi="figure")
|
|
106
|
+ plt.show()
|
|
107
|
+ plt.close()
|
|
108
|
+
|
|
109
|
+def generate_confusion_matrix():
|
|
110
|
+ kernels = ["linear", "poly", "rbf", "sigmoid"]
|
|
111
|
+ for kernel in kernels:
|
|
112
|
+ clsvm = SVC(kernel=kernel)
|
|
113
|
+ clsvm.fit(X=x_train, y=y_train)
|
|
114
|
+ plot_confusion_matrix(estimator=clsvm, X=x_test, y_true=y_test, values_format=".1f")
|
|
115
|
+ plt.show()
|
|
116
|
+
|
|
117
|
+def time_precision_recall_error():
|
|
118
|
+ kernels = ["linear", "poly", "rbf", "sigmoid"]
|
|
119
|
+ times = {}
|
|
120
|
+ scores = {}
|
|
121
|
+ recalls = {}
|
|
122
|
+ errors = {}
|
|
123
|
+ c_values = [0.1 + 0.18 * x for x in range(6)]
|
|
124
|
+ for kernel in kernels:
|
|
125
|
+ times[kernel] = {}
|
|
126
|
+ scores[kernel] = {}
|
|
127
|
+ recalls[kernel] = {}
|
|
128
|
+ errors[kernel] = {}
|
|
129
|
+ for c_value in c_values:
|
|
130
|
+ clsvm = SVC(kernel=kernel, C=c_value)
|
|
131
|
+ tic = time.process_time()
|
|
132
|
+ clsvm.fit(X=x_train, y=y_train)
|
|
133
|
+ toc = time.process_time()
|
|
134
|
+ y_pred = clsvm.predict(X=x_test)
|
|
135
|
+ times[kernel][c_value] = toc - tic
|
|
136
|
+ scores[kernel][c_value] = clsvm.score(X=x_test, y=y_test) * 100
|
|
137
|
+ recalls[kernel][c_value] = recall_score(y_true=y_test, y_pred=y_pred, average="micro")
|
|
138
|
+ errors[kernel][c_value] = 100 - scores[kernel][c_value]
|
|
139
|
+ print("Score avec kernel=%s: %.2f" % (kernel, clsvm.score(X=x_test, y=y_test) * 100))
|
|
140
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
141
|
+ plt.plot(c_values, list(times["linear"].values()), label="linear")
|
|
142
|
+ plt.plot(c_values, list(times["poly"].values()), label="poly")
|
|
143
|
+ plt.plot(c_values, list(times["rbf"].values()), label="rbf")
|
|
144
|
+ plt.plot(c_values, list(times["sigmoid"].values()), label="sigmoid")
|
|
145
|
+ plt.xlabel("Valeurs de C")
|
|
146
|
+ plt.xlim(left=0.1)
|
|
147
|
+ plt.ylabel("Temps d'apprentissage (secondes)")
|
|
148
|
+ plt.title("Courbe d'évolution du temps d'apprentissage en fonction de C")
|
|
149
|
+ plt.legend()
|
|
150
|
+ plt.savefig("tp3Output/times_all_kernel_with_c_variation.png", dpi="figure")
|
|
151
|
+ plt.show(); plt.close()
|
|
152
|
+ ###
|
|
153
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
154
|
+ plt.plot(c_values, list(scores["linear"].values()), label="linear precision")
|
|
155
|
+ plt.plot(c_values, list(scores["poly"].values()), label="poly precision")
|
|
156
|
+ plt.plot(c_values, list(scores["rbf"].values()), label="rbf precision")
|
|
157
|
+ plt.plot(c_values, list(scores["sigmoid"].values()), label="sigmoid precision")
|
|
158
|
+ plt.plot(c_values, list(errors["linear"].values()), label="linear error", marker="*", linestyle="-.")
|
|
159
|
+ plt.plot(c_values, list(errors["poly"].values()), label="poly error", marker="*", linestyle="-.")
|
|
160
|
+ plt.plot(c_values, list(errors["rbf"].values()), label="rbf error", marker="*", linestyle="-.")
|
|
161
|
+ plt.plot(c_values, list(errors["sigmoid"].values()), label="sigmoid error", marker="*", linestyle="-.")
|
|
162
|
+ plt.xlabel("Valeurs de C")
|
|
163
|
+ plt.xlim(left=0.1)
|
|
164
|
+ plt.ylabel("Score")
|
|
165
|
+ plt.title("Courbe d'évolution de la précision et de l'erreur de classification en fonction de C")
|
|
166
|
+ plt.legend()
|
|
167
|
+ plt.savefig("tp3Output/scores_and_errors_all_kernel_with_c_variation.png", dpi="figure")
|
|
168
|
+ plt.show(); plt.close()
|
|
169
|
+ ###
|
|
170
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
171
|
+ plt.plot(c_values, list(recalls["linear"].values()), label="linear")
|
|
172
|
+ plt.plot(c_values, list(recalls["poly"].values()), label="poly")
|
|
173
|
+ plt.plot(c_values, list(recalls["rbf"].values()), label="rbf")
|
|
174
|
+ plt.plot(c_values, list(recalls["sigmoid"].values()), label="sigmoid")
|
|
175
|
+ plt.xlabel("Valeurs de C")
|
|
176
|
+ plt.xlim(left=0.1)
|
|
177
|
+ plt.ylabel("Recall score")
|
|
178
|
+ plt.title("Courbe d'évolution du recall en fonction de C")
|
|
179
|
+ plt.legend()
|
|
180
|
+ plt.savefig("tp3Output/recalls_all_kernel_with_c_variation.png", dpi="figure")
|
|
181
|
+ plt.show(); plt.close()
|
|
182
|
+
|
|
183
|
+def methods_comparison():
|
|
184
|
+ times = {}
|
|
185
|
+ scores = {}
|
|
186
|
+ recalls = {}
|
|
187
|
+ errors = {}
|
|
188
|
+ matrices = {}
|
|
189
|
+ ### Méthode 1
|
|
190
|
+ knn = KNeighborsClassifier(n_neighbors=10)
|
|
191
|
+ tic = time.process_time()
|
|
192
|
+ knn.fit(X=x_train, y=y_train)
|
|
193
|
+ toc = time.process_time()
|
|
194
|
+ y_pred = knn.predict(X=x_test)
|
|
195
|
+ times["knn"] = toc - tic
|
|
196
|
+ scores["knn"] = knn.score(X=x_test, y=y_pred) * 100
|
|
197
|
+ recalls["knn"] = recall_score(y_true=y_test, y_pred=y_pred, average="micro")
|
|
198
|
+ errors["knn"] = 100 - scores["knn"]
|
|
199
|
+ # matrices["knn"] = confusion_matrix(y_true=y_test,y_pred=y_pred)
|
|
200
|
+ plot_confusion_matrix(estimator=knn, X=x_test, y_true=y_test, values_format=".1f"); plt.show()
|
|
201
|
+ ### Méthode 2
|
|
202
|
+ mlp = MLPClassifier(hidden_layer_sizes=(50))
|
|
203
|
+ tic = time.process_time()
|
|
204
|
+ mlp.fit(X=x_train, y=y_train)
|
|
205
|
+ toc = time.process_time()
|
|
206
|
+ y_pred = knn.predict(X=x_test)
|
|
207
|
+ times["mlp"] = toc - tic
|
|
208
|
+ scores["mlp"] = mlp.score(X=x_test, y=y_pred) * 100
|
|
209
|
+ recalls["mlp"] = recall_score(y_true=y_test, y_pred=y_pred, average="micro")
|
|
210
|
+ errors["mlp"] = 100 - scores["mlp"]
|
|
211
|
+ # matrices["mlp"] = confusion_matrix(y_true=y_test, y_pred=y_pred)
|
|
212
|
+ plot_confusion_matrix(estimator=mlp, X=x_test, y_true=y_test, values_format=".1f"); plt.show()
|
|
213
|
+ ### Méthode 3
|
|
214
|
+ svc = SVC(kernel="rbf", C=1)
|
|
215
|
+ tic = time.process_time()
|
|
216
|
+ svc.fit(X=x_train, y=y_train)
|
|
217
|
+ toc = time.process_time()
|
|
218
|
+ y_pred = knn.predict(X=x_test)
|
|
219
|
+ times["svc"] = toc - tic
|
|
220
|
+ scores["svc"] = svc.score(X=x_test, y=y_pred) * 100
|
|
221
|
+ recalls["svc"] = recall_score(y_true=y_test, y_pred=y_pred, average="micro")
|
|
222
|
+ errors["svc"] = 100 - scores["svc"]
|
|
223
|
+ # matrices["svc"] = confusion_matrix(y_true=y_test, y_pred=y_pred)
|
|
224
|
+ plot_confusion_matrix(estimator=svc, X=x_test, y_true=y_test, values_format=".1f"); plt.show()
|
|
225
|
+ ### Plotting
|
|
226
|
+ barWidth = 0.5
|
|
227
|
+ y1 = [times["knn"], times["mlp"], times["svc"]]
|
|
228
|
+ y2 = [scores["knn"], scores["mlp"], scores["svc"]]
|
|
229
|
+ y3 = [recalls["knn"], recalls["mlp"], recalls["svc"]]
|
|
230
|
+ y4 = [errors["knn"], errors["mlp"], errors["svc"]]
|
|
231
|
+ r1 = [0, 3, 6]
|
|
232
|
+ r2 = [x + barWidth for x in r1]
|
|
233
|
+ r3 = [x + 2 * barWidth for x in r1]
|
|
234
|
+ r4 = [x + 3 * barWidth for x in r1]
|
|
235
|
+ plt.figure(figsize=(12.8, 9.6))
|
|
236
|
+ bar1 = plt.bar(r1, y1, width=barWidth, color=['green' for i in y1], linewidth=2, label="temps (sec)")
|
|
237
|
+ bar2 = plt.bar(r2, y2, width=barWidth, color=['yellow' for i in y2], linewidth=4, label="precision (%)")
|
|
238
|
+ bar3 = plt.bar(r3, y3, width=barWidth, color=['red' for i in y3], linewidth=1, label="recall")
|
|
239
|
+ bar4 = plt.bar(r4, y4, width=barWidth, color=['magenta' for i in y4], linewidth=3, label="error (%)")
|
|
240
|
+ plt.xticks([r + 1.5 * barWidth for r in r1], ['Modèle K-nn', 'Modèle MLP', 'Modèle SVM'])
|
|
241
|
+ plt.xlabel("Méthodes")
|
|
242
|
+ plt.ylabel("Valeur")
|
|
243
|
+ plt.title("Evaluation de différentes métriques de performance pour chacune des méthodes vues")
|
|
244
|
+ for rect in bar1 + bar2 + bar3 + bar4:
|
|
245
|
+ height = rect.get_height()
|
|
246
|
+ plt.text(rect.get_x() + rect.get_width() / 2.0, height, '%.2f' % height, ha='center', va='bottom')
|
|
247
|
+ plt.legend()
|
|
248
|
+ plt.show()
|
|
249
|
+ plt.savefig("tp3Output/global_comparison_of_methods_different_metrics", dpi="figure")
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+kernel_variation()
|
|
253
|
+kernel_and_c_variation()
|
|
254
|
+train_test_error_with_c_variation()
|
|
255
|
+generate_confusion_matrix()
|
|
256
|
+time_precision_recall_error()
|
|
257
|
+methods_comparison()
|