0=Angry, 1=Disgust, 2=Fear, 3=Happy, 4=Sad, 5=Surprise, 6=Neutral
In [84]:
# display some images for every different expression
import numpy as np
import seaborn as sns
from keras.preprocessing.image import load_img, img_to_array
import matplotlib.pyplot as plt
import os
import pandas as pd
import sys
import cv2
from keras import models
from keras.utils import np_utils
from keras.callbacks import ModelCheckpoint, EarlyStopping
下載圖片
將下載的csv檔案轉換成圖片並分類
In [85]:
#訓練需要的圖片和label為csv形式，需要用以下程式碼將csv的內容轉變為圖片和label
data = np.genfromtxt('face_emition/fer2013/fer2013.csv',delimiter=',',dtype=None)
#第0欄為label，第1欄為image的string形式
labels = data[1:,0].astype(np.int32)
image_buffer = data[1:,1]
#把string形式的image變成uint8的格式，最後轉變為numpy array
images = np.array([np.fromstring(image, np.uint8, sep=' ') for image in image_buffer])
#usage為訓練或測試資料的標記
usage = data[1:,2]
#將labels, images, usages以zip的形式打包成一個個的tuple
dataset = zip(labels, images, usage)
#label的數字轉文字的翻譯
label_names = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
emotion_dict = dict((j,i) for i,j in enumerate(label_names))
inverse_emotion_dict = dict((i,j) for i,j in enumerate(label_names))
output_path = os.getcwd()
df=dataset
#以下for迴圈把image array儲存成.jpg的格式並且分別儲存在training/test下，且又依據不同的label分開存放
for i, d in enumerate(dataset):
path = os.path.join(output_path, 'face_emition/data')
usage_path = os.path.join(path, eval(str(d[-1]).lstrip('b')))
img = d[1].reshape((48,48))
label_path = os.path.join(usage_path, label_names[d[0]])
if not os.path.isdir(usage_path):
os.mkdir(usage_path)
img_name = '%08d.jpg' % i
img_path = os.path.join(label_path, img_name)
if not os.path.isdir(label_path):
os.mkdir(label_path)
cv2.imwrite(img_path, img)
/Users/laihunglai/anaconda3/envs/py35/lib/python3.5/site-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Reading unicode strings without specifying the encoding argument is deprecated. Set the encoding, use None for the system default.
將圖片從讀入資料架中讀入，label為資料夾名稱
In [86]:
output_path = os.getcwd()
path = os.path.join(output_path, 'face_emition/data')
data_list = os.listdir(path)
img_data=[]
emotion_data=[]
for i in data_list:
file_path = os.path.join(path, i)
file_list = os.listdir(file_path)
for j in file_list:
file_n_path = os.path.join(file_path, j)
file_n_list = os.listdir(file_n_path)
for k in file_n_list:
figure_path = os.path.join(file_n_path,k)
img = cv2.imread(figure_path)
img_data.append(img)
emotion_data.append(emotion_dict[j])
In [88]:
#顯示讀入幾筆訓練資料
print('total number of data:', len(img_data))
#將訓練資料轉成numpy array
emotion_data = np.array(emotion_data)
img_data = np.array(img_data)
print('shape of data:', img_data.shape)
total number of data: 35887
shape of data: (35887, 48, 48, 3)
In [89]:
#將data分成訓練和測試資料
mask = np.random.rand(len(emotion_data)) #隨機產生一個1D mask用來隨機篩選traning和test資料
trainX = img_data[mask>0.8]
trainY = emotion_data[mask>0.8]
testX = img_data[mask<0.2]
testY = emotion_data[mask<0.2]
#normalizetraning set的pixel到[0,1]
trainX_normalize=trainX/225
testX_normalize=testX/225
In [90]:
#label的資料預處理:利用One-hot enconding將0~6的數字轉換成7個0或1的組合,對應到7個神經元
trainY_OneHot=np_utils.to_categorical(trainY)
testY_OneHot=np_utils.to_categorical(testY)
In [91]:
from keras.models import Sequential
from keras.layers import Dense, Conv2D,MaxPooling2D,Dropout,Flatten
#建立Sequential模型
model=Sequential()
model.add(Conv2D(filters=8,kernel_size=(3,3),padding='same',input_shape=(48,48,3),activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(filters=16,kernel_size=(3,3),padding='same',activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(32,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7,activation='softmax'))
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_5 (Conv2D) (None, 48, 48, 8) 224
_________________________________________________________________
max_pooling2d_5 (MaxPooling2 (None, 24, 24, 8) 0
_________________________________________________________________
conv2d_6 (Conv2D) (None, 24, 24, 16) 1168
_________________________________________________________________
max_pooling2d_6 (MaxPooling2 (None, 12, 12, 16) 0
_________________________________________________________________
dropout_5 (Dropout) (None, 12, 12, 16) 0
_________________________________________________________________
flatten_3 (Flatten) (None, 2304) 0
_________________________________________________________________
dense_5 (Dense) (None, 32) 73760
_________________________________________________________________
dropout_6 (Dropout) (None, 32) 0
_________________________________________________________________
dense_6 (Dense) (None, 7) 231
=================================================================
Total params: 75,383
Trainable params: 75,383
Non-trainable params: 0
_________________________________________________________________
In [71]:
model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
#進行訓練
es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=5)
mc = ModelCheckpoint('emotion.h5', monitor='val_loss', mode='min', verbose=2, save_best_only=True)
train_history=model.fit(x=trainX_normalize,y=trainY_OneHot, validation_split=0.2, epochs=50, batch_size=256, verbose=2, callbacks=[es, mc])
Train on 5716 samples, validate on 1430 samples
Epoch 1/50
- 111s - loss: 1.8174 - acc: 0.2474 - val_loss: 2.3562 - val_acc: 0.1147
Epoch 00001: val_loss improved from inf to 2.35617, saving model to emotion.h5
Epoch 2/50
- 85s - loss: 1.7631 - acc: 0.2724 - val_loss: 2.3993 - val_acc: 0.1147
Epoch 00002: val_loss did not improve from 2.35617
Epoch 3/50
- 83s - loss: 1.7386 - acc: 0.2720 - val_loss: 2.4787 - val_acc: 0.1147
Epoch 00003: val_loss did not improve from 2.35617
Epoch 4/50
- 91s - loss: 1.7025 - acc: 0.2953 - val_loss: 2.4697 - val_acc: 0.1175
Epoch 00004: val_loss did not improve from 2.35617
Epoch 5/50
- 91s - loss: 1.6702 - acc: 0.3104 - val_loss: 2.5253 - val_acc: 0.1455
Epoch 00005: val_loss did not improve from 2.35617
Epoch 6/50
- 99s - loss: 1.6370 - acc: 0.3188 - val_loss: 2.3106 - val_acc: 0.1580
Epoch 00006: val_loss improved from 2.35617 to 2.31061, saving model to emotion.h5
Epoch 7/50
- 86s - loss: 1.6132 - acc: 0.3401 - val_loss: 2.6427 - val_acc: 0.1469
Epoch 00007: val_loss did not improve from 2.31061
Epoch 8/50
- 83s - loss: 1.5911 - acc: 0.3606 - val_loss: 2.6162 - val_acc: 0.1671
Epoch 00008: val_loss did not improve from 2.31061
Epoch 9/50
- 84s - loss: 1.5665 - acc: 0.3690 - val_loss: 2.5161 - val_acc: 0.1748
Epoch 00009: val_loss did not improve from 2.31061
Epoch 10/50
- 71s - loss: 1.5497 - acc: 0.3796 - val_loss: 2.5572 - val_acc: 0.1797
Epoch 00010: val_loss did not improve from 2.31061
Epoch 11/50
- 75s - loss: 1.5366 - acc: 0.3907 - val_loss: 2.6521 - val_acc: 0.1727
Epoch 00011: val_loss did not improve from 2.31061
在 Keras 中若要儲存與載入訓練好的模型或參數，可以使用其內建模型儲存與載入功能，將模型儲存於 HDF5(.h5) 或 JSON(.json) 檔案中，以下是 Keras 儲存模型的操作方式。
上面訓練結束自動儲存的emotion.h5內含模型和權重參數下次要使用時載入的指令如下
In [76]:
# 從 HDF5 檔案中載入模型
model = models.load_model('emotion.h5')
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_3 (Conv2D) (None, 48, 48, 8) 224
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 24, 24, 8) 0
_________________________________________________________________
conv2d_4 (Conv2D) (None, 24, 24, 16) 1168
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 12, 12, 16) 0
_________________________________________________________________
dropout_3 (Dropout) (None, 12, 12, 16) 0
_________________________________________________________________
flatten_2 (Flatten) (None, 2304) 0
_________________________________________________________________
dense_3 (Dense) (None, 32) 73760
_________________________________________________________________
dropout_4 (Dropout) (None, 32) 0
_________________________________________________________________
dense_4 (Dense) (None, 7) 231
=================================================================
Total params: 75,383
Trainable params: 75,383
Non-trainable params: 0
_________________________________________________________________
以下為其他權重和模型儲存方式，這個案例沒有使用
如果只要將模型儲存起來，不儲存其中的參數，可以使用 to_json 或 to_yaml 將模型轉為 JSON 或 YAML 的文字資料，在自己儲存至檔案中：
In [73]:
# 將模型匯出至 JSON（不含參數）
json_emotion = model.to_json()
# 從 JSON 資料重建模型
model = models.model_from_json(json_emotion)
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_3 (Conv2D) (None, 48, 48, 8) 224
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 24, 24, 8) 0
_________________________________________________________________
conv2d_4 (Conv2D) (None, 24, 24, 16) 1168
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 12, 12, 16) 0
_________________________________________________________________
dropout_3 (Dropout) (None, 12, 12, 16) 0
_________________________________________________________________
flatten_2 (Flatten) (None, 2304) 0
_________________________________________________________________
dense_3 (Dense) (None, 32) 73760
_________________________________________________________________
dropout_4 (Dropout) (None, 32) 0
_________________________________________________________________
dense_4 (Dense) (None, 7) 231
=================================================================
Total params: 75,383
Trainable params: 75,383
Non-trainable params: 0
_________________________________________________________________
若只想要儲存模型的參數（也就是 weights），不包含模型本身，可以使用 save_weights：
In [74]:
# 將參數儲存至 HDF5 檔案（不含模型）
model.save_weights('emotion_weights.h5')
# 從 HDF5 檔案載入參數（不含模型）
model.load_weights('emotion_weights.h5')
若要將儲存的參數載入至不同的模型中使用（模型不同，但有相同網路層，例如 fine-tuning 或 transfer-learning），可以加上 by_name 參數：
In [75]:
# 載入參數至不同的模型中使用
model.load_weights('emotion_weights.h5', by_name = True)
In [77]:
#視覺化訓練過程
import matplotlib.pyplot as plt
def show_train_history(train_history,train,validation):
plt.plot(train_history.history[train])
plt.plot(train_history.history[validation])
plt.title('Train History')
plt.ylabel(train)
plt.xlabel('Epoch')
plt.legend(['train','validation'],loc='upper left')
plt.show()
show_train_history(train_history,'acc','val_acc')
#若訓練(train)的準確度一直增加而驗證(validation)的準確度沒有一直增加則可能是overfit
#畫出loss誤差執行結果
show_train_history(train_history,'loss','val_loss')
#測試model的準確度
score=model.evaluate(testX_normalize,testY_OneHot)
print('accuracy=',score[1])
#進行預測
prediction=model.predict_classes(testX_normalize)
7263/7263 [==============================] - 50s 7ms/step
accuracy= 0.3224562852815641
In [78]:
import matplotlib.pyplot as plt
def plot_images_labels_prediction(images,labels,prediction,idx,num=10):
#images: 影像; labels: 答案; prediction: 預測結果; idx: 開始顯示的資料; num: 要顯示的資料筆數
fig=plt.gcf()
fig.set_size_inches(12,14)
if num>25:num=25
for i in range(0,num):
ax=plt.subplot(5,5,1+i)
ax.imshow(images[idx])
title='label='+str(inverse_emotion_dict[labels[idx]])
if len(prediction)>0:
title+=",predict="+str(inverse_emotion_dict[prediction[idx]])
ax.set_title(title,fontsize=8)
ax.set_xticks([]);ax.set_yticks([])
idx+=1
plt.show()
In [79]:
#顯示前10筆預測結果, 顯示的200筆到209筆共20筆data
plot_images_labels_prediction(testX,testY,prediction,200,100)
#用pandas crosstab建立混淆矩陣
import pandas
import seaborn as sns
heat = pd.crosstab(testY, prediction,rownames=['label'],colnames=['prediction'])
sns.heatmap(heat)
Out[79]:
<matplotlib.axes._subplots.AxesSubplot at 0x1a7969c9b0>
對視訊鏡頭的影像進行表情辨識
import cv2
filename = '../../people.png'
face_cascade =cv2.CascadeClassifier('../../cascades/haarcascade_frontalface_default.xml')
img = cv2.imread(filename)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.3, 5)
for (x,y,w,h) in faces:
img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
cv2.namedWindow('human Detected!!')
cv2.imshow('human Detected!!', img)
cv2.imwrite('./human.jpg', img)
cv2.waitKey(0)
In [ ]:
import cv2
from keras import models
import numpy as np
# 從 HDF5 檔案中載入模型
model = models.load_model('../../../emotion.h5')
label_names = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
inverse_emotion_dict = dict((i,j) for i,j in enumerate(label_names))
face_cascade =cv2.CascadeClassifier('../../cascades/haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier('../../cascades/haarcascade_eye.xml')
camera = cv2.VideoCapture(1)
while (True):
ret, frame = camera.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
color=frame
faces = face_cascade.detectMultiScale(gray, 1.3, 5)
for (x,y,w,h) in faces:
img = cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
roi_gray = gray[y:y+h, x:x+w]
roi_color = color[y:y+h, x:x+w]
face_img=cv2.resize(roi_color,(48,48))
face_img=face_img/255
prediction=model.predict_classes(face_img[np.newaxis,:])
emotion= [inverse_emotion_dict[i] for i in prediction]
cv2.putText(img,emotion[0],(x,y-20),cv2.FONT_HERSHEY_SIMPLEX,1,255,2)
eyes = eye_cascade.detectMultiScale(roi_gray, 1.03, 5, 0,(40,40))#最小搜尋眼睛的pixel size: 40*40 pixels
for (ex,ey,ew,eh) in eyes:
cv2.rectangle(img,(x+ex,y+ey),(x+ex+ew,y+ey+eh),(0,255,0),2)
cv2.imshow("camera", frame)
if cv2.waitKey(1000 // 12) & 0xff == ord("q"):
break
camera.release()
cv2.destroyAllWindows()
/Users/laihunglai/anaconda3/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
from ._conv import register_converters as _register_converters
Using TensorFlow backend.
