DeepFaceLab/facelib/LandmarksProcessor.py
2021-10-18 10:51:25 +04:00

900 lines
32 KiB
Python

import colorsys
import math
from enum import IntEnum
import cv2
import numpy as np
import numpy.linalg as npla
from core import imagelib
from core import mathlib
from facelib import FaceType
from core.mathlib.umeyama import umeyama
landmarks_2D = np.array([
[ 0.000213256, 0.106454 ], #17
[ 0.0752622, 0.038915 ], #18
[ 0.18113, 0.0187482 ], #19
[ 0.29077, 0.0344891 ], #20
[ 0.393397, 0.0773906 ], #21
[ 0.586856, 0.0773906 ], #22
[ 0.689483, 0.0344891 ], #23
[ 0.799124, 0.0187482 ], #24
[ 0.904991, 0.038915 ], #25
[ 0.98004, 0.106454 ], #26
[ 0.490127, 0.203352 ], #27
[ 0.490127, 0.307009 ], #28
[ 0.490127, 0.409805 ], #29
[ 0.490127, 0.515625 ], #30
[ 0.36688, 0.587326 ], #31
[ 0.426036, 0.609345 ], #32
[ 0.490127, 0.628106 ], #33
[ 0.554217, 0.609345 ], #34
[ 0.613373, 0.587326 ], #35
[ 0.121737, 0.216423 ], #36
[ 0.187122, 0.178758 ], #37
[ 0.265825, 0.179852 ], #38
[ 0.334606, 0.231733 ], #39
[ 0.260918, 0.245099 ], #40
[ 0.182743, 0.244077 ], #41
[ 0.645647, 0.231733 ], #42
[ 0.714428, 0.179852 ], #43
[ 0.793132, 0.178758 ], #44
[ 0.858516, 0.216423 ], #45
[ 0.79751, 0.244077 ], #46
[ 0.719335, 0.245099 ], #47
[ 0.254149, 0.780233 ], #48
[ 0.340985, 0.745405 ], #49
[ 0.428858, 0.727388 ], #50
[ 0.490127, 0.742578 ], #51
[ 0.551395, 0.727388 ], #52
[ 0.639268, 0.745405 ], #53
[ 0.726104, 0.780233 ], #54
[ 0.642159, 0.864805 ], #55
[ 0.556721, 0.902192 ], #56
[ 0.490127, 0.909281 ], #57
[ 0.423532, 0.902192 ], #58
[ 0.338094, 0.864805 ], #59
[ 0.290379, 0.784792 ], #60
[ 0.428096, 0.778746 ], #61
[ 0.490127, 0.785343 ], #62
[ 0.552157, 0.778746 ], #63
[ 0.689874, 0.784792 ], #64
[ 0.553364, 0.824182 ], #65
[ 0.490127, 0.831803 ], #66
[ 0.42689 , 0.824182 ] #67
], dtype=np.float32)
landmarks_2D_new = np.array([
[ 0.000213256, 0.106454 ], #17
[ 0.0752622, 0.038915 ], #18
[ 0.18113, 0.0187482 ], #19
[ 0.29077, 0.0344891 ], #20
[ 0.393397, 0.0773906 ], #21
[ 0.586856, 0.0773906 ], #22
[ 0.689483, 0.0344891 ], #23
[ 0.799124, 0.0187482 ], #24
[ 0.904991, 0.038915 ], #25
[ 0.98004, 0.106454 ], #26
[ 0.490127, 0.203352 ], #27
[ 0.490127, 0.307009 ], #28
[ 0.490127, 0.409805 ], #29
[ 0.490127, 0.515625 ], #30
[ 0.36688, 0.587326 ], #31
[ 0.426036, 0.609345 ], #32
[ 0.490127, 0.628106 ], #33
[ 0.554217, 0.609345 ], #34
[ 0.613373, 0.587326 ], #35
[ 0.121737, 0.216423 ], #36
[ 0.187122, 0.178758 ], #37
[ 0.265825, 0.179852 ], #38
[ 0.334606, 0.231733 ], #39
[ 0.260918, 0.245099 ], #40
[ 0.182743, 0.244077 ], #41
[ 0.645647, 0.231733 ], #42
[ 0.714428, 0.179852 ], #43
[ 0.793132, 0.178758 ], #44
[ 0.858516, 0.216423 ], #45
[ 0.79751, 0.244077 ], #46
[ 0.719335, 0.245099 ], #47
[ 0.254149, 0.780233 ], #48
[ 0.726104, 0.780233 ], #54
], dtype=np.float32)
mouth_center_landmarks_2D = np.array([
[-4.4202591e-07, 4.4916576e-01], #48
[ 1.8399176e-01, 3.7537053e-01], #49
[ 3.7018123e-01, 3.3719531e-01], #50
[ 5.0000089e-01, 3.6938059e-01], #51
[ 6.2981832e-01, 3.3719531e-01], #52
[ 8.1600773e-01, 3.7537053e-01], #53
[ 1.0000000e+00, 4.4916576e-01], #54
[ 8.2213330e-01, 6.2836081e-01], #55
[ 6.4110327e-01, 7.0757812e-01], #56
[ 5.0000089e-01, 7.2259867e-01], #57
[ 3.5889623e-01, 7.0757812e-01], #58
[ 1.7786618e-01, 6.2836081e-01], #59
[ 7.6765373e-02, 4.5882553e-01], #60
[ 3.6856663e-01, 4.4601500e-01], #61
[ 5.0000089e-01, 4.5999300e-01], #62
[ 6.3143289e-01, 4.4601500e-01], #63
[ 9.2323411e-01, 4.5882553e-01], #64
[ 6.3399029e-01, 5.4228687e-01], #65
[ 5.0000089e-01, 5.5843467e-01], #66
[ 3.6601129e-01, 5.4228687e-01] #67
], dtype=np.float32)
# 68 point landmark definitions
landmarks_68_pt = { "mouth": (48,68),
"right_eyebrow": (17, 22),
"left_eyebrow": (22, 27),
"right_eye": (36, 42),
"left_eye": (42, 48),
"nose": (27, 36), # missed one point
"jaw": (0, 17) }
landmarks_68_3D = np.array( [
[-73.393523 , -29.801432 , 47.667532 ], #00
[-72.775014 , -10.949766 , 45.909403 ], #01
[-70.533638 , 7.929818 , 44.842580 ], #02
[-66.850058 , 26.074280 , 43.141114 ], #03
[-59.790187 , 42.564390 , 38.635298 ], #04
[-48.368973 , 56.481080 , 30.750622 ], #05
[-34.121101 , 67.246992 , 18.456453 ], #06
[-17.875411 , 75.056892 , 3.609035 ], #07
[0.098749 , 77.061286 , -0.881698 ], #08
[17.477031 , 74.758448 , 5.181201 ], #09
[32.648966 , 66.929021 , 19.176563 ], #10
[46.372358 , 56.311389 , 30.770570 ], #11
[57.343480 , 42.419126 , 37.628629 ], #12
[64.388482 , 25.455880 , 40.886309 ], #13
[68.212038 , 6.990805 , 42.281449 ], #14
[70.486405 , -11.666193 , 44.142567 ], #15
[71.375822 , -30.365191 , 47.140426 ], #16
[-61.119406 , -49.361602 , 14.254422 ], #17
[-51.287588 , -58.769795 , 7.268147 ], #18
[-37.804800 , -61.996155 , 0.442051 ], #19
[-24.022754 , -61.033399 , -6.606501 ], #20
[-11.635713 , -56.686759 , -11.967398 ], #21
[12.056636 , -57.391033 , -12.051204 ], #22
[25.106256 , -61.902186 , -7.315098 ], #23
[38.338588 , -62.777713 , -1.022953 ], #24
[51.191007 , -59.302347 , 5.349435 ], #25
[60.053851 , -50.190255 , 11.615746 ], #26
[0.653940 , -42.193790 , -13.380835 ], #27
[0.804809 , -30.993721 , -21.150853 ], #28
[0.992204 , -19.944596 , -29.284036 ], #29
[1.226783 , -8.414541 , -36.948060 ], #00
[-14.772472 , 2.598255 , -20.132003 ], #01
[-7.180239 , 4.751589 , -23.536684 ], #02
[0.555920 , 6.562900 , -25.944448 ], #03
[8.272499 , 4.661005 , -23.695741 ], #04
[15.214351 , 2.643046 , -20.858157 ], #05
[-46.047290 , -37.471411 , 7.037989 ], #06
[-37.674688 , -42.730510 , 3.021217 ], #07
[-27.883856 , -42.711517 , 1.353629 ], #08
[-19.648268 , -36.754742 , -0.111088 ], #09
[-28.272965 , -35.134493 , -0.147273 ], #10
[-38.082418 , -34.919043 , 1.476612 ], #11
[19.265868 , -37.032306 , -0.665746 ], #12
[27.894191 , -43.342445 , 0.247660 ], #13
[37.437529 , -43.110822 , 1.696435 ], #14
[45.170805 , -38.086515 , 4.894163 ], #15
[38.196454 , -35.532024 , 0.282961 ], #16
[28.764989 , -35.484289 , -1.172675 ], #17
[-28.916267 , 28.612716 , -2.240310 ], #18
[-17.533194 , 22.172187 , -15.934335 ], #19
[-6.684590 , 19.029051 , -22.611355 ], #20
[0.381001 , 20.721118 , -23.748437 ], #21
[8.375443 , 19.035460 , -22.721995 ], #22
[18.876618 , 22.394109 , -15.610679 ], #23
[28.794412 , 28.079924 , -3.217393 ], #24
[19.057574 , 36.298248 , -14.987997 ], #25
[8.956375 , 39.634575 , -22.554245 ], #26
[0.381549 , 40.395647 , -23.591626 ], #27
[-7.428895 , 39.836405 , -22.406106 ], #28
[-18.160634 , 36.677899 , -15.121907 ], #29
[-24.377490 , 28.677771 , -4.785684 ], #30
[-6.897633 , 25.475976 , -20.893742 ], #31
[0.340663 , 26.014269 , -22.220479 ], #32
[8.444722 , 25.326198 , -21.025520 ], #33
[24.474473 , 28.323008 , -5.712776 ], #34
[8.449166 , 30.596216 , -20.671489 ], #35
[0.205322 , 31.408738 , -21.903670 ], #36
[-7.198266 , 30.844876 , -20.328022 ] #37
], dtype=np.float32)
FaceType_to_padding_remove_align = {
FaceType.HALF: (0.0, False),
FaceType.MID_FULL: (0.0675, False),
FaceType.FULL: (0.2109375, False),
FaceType.FULL_NO_ALIGN: (0.2109375, True),
FaceType.WHOLE_FACE: (0.40, False),
FaceType.HEAD: (0.70, False),
FaceType.HEAD_NO_ALIGN: (0.70, True),
}
def convert_98_to_68(lmrks):
#jaw
result = [ lmrks[0] ]
for i in range(2,16,2):
result += [ ( lmrks[i] + (lmrks[i-1]+lmrks[i+1])/2 ) / 2 ]
result += [ lmrks[16] ]
for i in range(18,32,2):
result += [ ( lmrks[i] + (lmrks[i-1]+lmrks[i+1])/2 ) / 2 ]
result += [ lmrks[32] ]
#eyebrows averaging
result += [ lmrks[33],
(lmrks[34]+lmrks[41])/2,
(lmrks[35]+lmrks[40])/2,
(lmrks[36]+lmrks[39])/2,
(lmrks[37]+lmrks[38])/2,
]
result += [ (lmrks[42]+lmrks[50])/2,
(lmrks[43]+lmrks[49])/2,
(lmrks[44]+lmrks[48])/2,
(lmrks[45]+lmrks[47])/2,
lmrks[46]
]
#nose
result += list ( lmrks[51:60] )
#left eye (from our view)
result += [ lmrks[60],
lmrks[61],
lmrks[63],
lmrks[64],
lmrks[65],
lmrks[67] ]
#right eye
result += [ lmrks[68],
lmrks[69],
lmrks[71],
lmrks[72],
lmrks[73],
lmrks[75] ]
#mouth
result += list ( lmrks[76:96] )
return np.concatenate (result).reshape ( (68,2) )
def transform_points(points, mat, invert=False):
if invert:
mat = cv2.invertAffineTransform (mat)
points = np.expand_dims(points, axis=1)
points = cv2.transform(points, mat, points.shape)
points = np.squeeze(points)
return points
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
# get corner points in global space
g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True)
g_c = g_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (g_p[2]-g_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (g_p[1]-g_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
mod = (1.0 / scale)* ( npla.norm(g_p[0]-g_p[2])*(padding*np.sqrt(2.0) + 0.5) )
if face_type == FaceType.WHOLE_FACE:
# adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead
vec = (g_p[0]-g_p[3]).astype(np.float32)
vec_len = npla.norm(vec)
vec /= vec_len
g_c += vec*vec_len*0.07
elif face_type == FaceType.HEAD:
# assuming image_landmarks are 3D_Landmarks extracted for HEAD,
# adjust horizontal offset according to estimated yaw
yaw = estimate_averaged_yaw(transform_points (image_landmarks, mat, False))
hvec = (g_p[0]-g_p[1]).astype(np.float32)
hvec_len = npla.norm(hvec)
hvec /= hvec_len
yaw *= np.abs(math.tanh(yaw*2)) # Damp near zero
g_c -= hvec * (yaw * hvec_len / 2.0)
# adjust vertical offset for HEAD, 50% below
vvec = (g_p[0]-g_p[3]).astype(np.float32)
vvec_len = npla.norm(vvec)
vvec /= vvec_len
g_c += vvec*vvec_len*0.50
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array( [ g_c - tb_diag_vec*mod,
g_c + bt_diag_vec*mod,
g_c + tb_diag_vec*mod ] )
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array( [ g_c - tb_diag_vec*mod,
g_c + bt_diag_vec*mod,
g_c + tb_diag_vec*mod,
g_c - bt_diag_vec*mod,
] )
# get area of face square in global space
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array( [ g_c + [-side,-side],
g_c + [ side,-side],
g_c + [ side, side] ] )
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
return mat
def get_rect_from_landmarks(image_landmarks):
mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN)
g_p = transform_points ( np.float32([(0,0),(255,255) ]) , mat, True)
(l,t,r,b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1]
return (l,t,r,b)
def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0):
if len(lmrks) != 68:
raise Exception('works only with 68 landmarks')
lmrks = np.array( lmrks.copy(), dtype=np.int )
# #nose
ml_pnt = (lmrks[36] + lmrks[0]) // 2
mr_pnt = (lmrks[16] + lmrks[45]) // 2
# mid points between the mid points and eye
ql_pnt = (lmrks[36] + ml_pnt) // 2
qr_pnt = (lmrks[45] + mr_pnt) // 2
# Top of the eye arrays
bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39]))
bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt))
# Eyebrow arrays
top_l = lmrks[17:22]
top_r = lmrks[22:27]
# Adjust eyebrow arrays
lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l)
lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r)
return lmrks
def get_image_hull_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0 ):
hull_mask = np.zeros(image_shape[0:2]+(1,),dtype=np.float32)
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
r_jaw = (lmrks[0:9], lmrks[17:18])
l_jaw = (lmrks[8:17], lmrks[26:27])
r_cheek = (lmrks[17:20], lmrks[8:9])
l_cheek = (lmrks[24:27], lmrks[8:9])
nose_ridge = (lmrks[19:25], lmrks[8:9],)
r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9])
l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9])
nose = (lmrks[27:31], lmrks[31:36])
parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
for item in parts:
merged = np.concatenate(item)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,) )
return hull_mask
def get_image_eye_mask (image_shape, image_landmarks):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
h,w,c = image_shape
hull_mask = np.zeros( (h,w,1),dtype=np.float32)
image_landmarks = image_landmarks.astype(np.int)
cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[36:42]), (1,) )
cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[42:48]), (1,) )
dilate = h // 32
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(dilate,dilate)), iterations = 1 )
blur = h // 16
blur = blur + (1-blur % 2)
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur) , 0)
hull_mask = hull_mask[...,None]
return hull_mask
def get_image_mouth_mask (image_shape, image_landmarks):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
h,w,c = image_shape
hull_mask = np.zeros( (h,w,1),dtype=np.float32)
image_landmarks = image_landmarks.astype(np.int)
cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[60:]), (1,) )
dilate = h // 32
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(dilate,dilate)), iterations = 1 )
blur = h // 16
blur = blur + (1-blur % 2)
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur) , 0)
hull_mask = hull_mask[...,None]
return hull_mask
def alpha_to_color (img_alpha, color):
if len(img_alpha.shape) == 2:
img_alpha = img_alpha[...,None]
h,w,c = img_alpha.shape
result = np.zeros( (h,w, len(color) ), dtype=np.float32 )
result[:,:] = color
return result * img_alpha
def get_cmask (image_shape, lmrks, eyebrows_expand_mod=1.0):
h,w,c = image_shape
hull = get_image_hull_mask (image_shape, lmrks, eyebrows_expand_mod )
result = np.zeros( (h,w,3), dtype=np.float32 )
def process(w,h, data ):
d = {}
cur_lc = 0
all_lines = []
for s, pts_loop_ar in data:
lines = []
for pts, loop in pts_loop_ar:
pts_len = len(pts)
lines.append ( [ [ pts[i], pts[(i+1) % pts_len ] ] for i in range(pts_len - (0 if loop else 1) ) ] )
lines = np.concatenate (lines)
lc = lines.shape[0]
all_lines.append(lines)
d[s] = cur_lc, cur_lc+lc
cur_lc += lc
all_lines = np.concatenate (all_lines, 0)
#calculate signed distance for all points and lines
line_count = all_lines.shape[0]
pts_count = w*h
all_lines = np.repeat ( all_lines[None,...], pts_count, axis=0 ).reshape ( (pts_count*line_count,2,2) )
pts = np.empty( (h,w,line_count,2), dtype=np.float32 )
pts[...,1] = np.arange(h)[:,None,None]
pts[...,0] = np.arange(w)[:,None]
pts = pts.reshape ( (h*w*line_count, -1) )
a = all_lines[:,0,:]
b = all_lines[:,1,:]
pa = pts-a
ba = b-a
ph = np.clip ( np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1 )
dists = npla.norm ( pa - ba*ph[...,None], axis=1).reshape ( (h,w,line_count) )
def get_dists(name, thickness=0):
s,e = d[name]
result = dists[...,s:e]
if thickness != 0:
result = np.abs(result)-thickness
return np.min (result, axis=-1)
return get_dists
l_eye = lmrks[42:48]
r_eye = lmrks[36:42]
l_brow = lmrks[22:27]
r_brow = lmrks[17:22]
mouth = lmrks[48:60]
up_nose = np.concatenate( (lmrks[27:31], lmrks[33:34]) )
down_nose = lmrks[31:36]
nose = np.concatenate ( (up_nose, down_nose) )
gdf = process ( w,h,
(
('eyes', ((l_eye, True), (r_eye, True)) ),
('brows', ((l_brow, False), (r_brow,False)) ),
('up_nose', ((up_nose, False),) ),
('down_nose', ((down_nose, False),) ),
('mouth', ((mouth, True),) ),
)
)
eyes_fall_dist = w // 32
eyes_thickness = max( w // 64, 1 )
brows_fall_dist = w // 32
brows_thickness = max( w // 256, 1 )
nose_fall_dist = w / 12
nose_thickness = max( w // 96, 1 )
mouth_fall_dist = w // 32
mouth_thickness = max( w // 64, 1 )
eyes_mask = gdf('eyes',eyes_thickness)
eyes_mask = 1-np.clip( eyes_mask/ eyes_fall_dist, 0, 1)
#eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
#eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
brows_mask = gdf('brows', brows_thickness)
brows_mask = 1-np.clip( brows_mask / brows_fall_dist, 0, 1)
#brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1)
mouth_mask = gdf('mouth', mouth_thickness)
mouth_mask = 1-np.clip( mouth_mask / mouth_fall_dist, 0, 1)
#mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1)
def blend(a,b,k):
x = np.clip ( 0.5+0.5*(b-a)/k, 0.0, 1.0 )
return (a-b)*x+b - k*x*(1.0-x)
#nose_mask = (a-b)*x+b - k*x*(1.0-x)
#nose_mask = np.minimum (up_nose_mask , down_nose_mask )
#nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
nose_mask = blend ( gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness*3 )
nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
up_nose_mask = gdf('up_nose', nose_thickness)
up_nose_mask = 1-np.clip( up_nose_mask / nose_fall_dist, 0, 1)
#up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1)
down_nose_mask = gdf('down_nose', nose_thickness)
down_nose_mask = 1-np.clip( down_nose_mask / nose_fall_dist, 0, 1)
#down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1)
#nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 )
#nose_mask /= np.max(nose_mask)
#nose_mask = np.maximum (up_nose_mask , down_nose_mask )
#nose_mask = down_nose_mask
#nose_mask = np.zeros_like(nose_mask)
eyes_mask = eyes_mask * (1-mouth_mask)
nose_mask = nose_mask * (1-eyes_mask)
hull_mask = hull[...,0].copy()
hull_mask = hull_mask * (1-eyes_mask) * (1-brows_mask) * (1-nose_mask) * (1-mouth_mask)
#eyes_mask = eyes_mask * (1-nose_mask)
mouth_mask= mouth_mask * (1-nose_mask)
brows_mask = brows_mask * (1-nose_mask)* (1-eyes_mask )
hull_mask = alpha_to_color(hull_mask, (0,1,0) )
eyes_mask = alpha_to_color(eyes_mask, (1,0,0) )
brows_mask = alpha_to_color(brows_mask, (0,0,1) )
nose_mask = alpha_to_color(nose_mask, (0,1,1) )
mouth_mask = alpha_to_color(mouth_mask, (0,0,1) )
#nose_mask = np.maximum( up_nose_mask, down_nose_mask )
result = hull_mask + mouth_mask+ nose_mask + brows_mask + eyes_mask
result *= hull
#result = np.clip (result, 0, 1)
return result
def blur_image_hull_mask (hull_mask):
maxregion = np.argwhere(hull_mask==1.0)
miny,minx = maxregion.min(axis=0)[:2]
maxy,maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx;
leny = maxy - miny;
masky = int(minx+(lenx//2))
maskx = int(miny+(leny//2))
lowest_len = min (lenx, leny)
ero = int( lowest_len * 0.085 )
blur = int( lowest_len * 0.10 )
hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 )
hull_mask = cv2.blur(hull_mask, (blur, blur) )
hull_mask = np.expand_dims (hull_mask,-1)
return hull_mask
mirror_idxs = [
[0,16],
[1,15],
[2,14],
[3,13],
[4,12],
[5,11],
[6,10],
[7,9],
[17,26],
[18,25],
[19,24],
[20,23],
[21,22],
[36,45],
[37,44],
[38,43],
[39,42],
[40,47],
[41,46],
[31,35],
[32,34],
[50,52],
[49,53],
[48,54],
[59,55],
[58,56],
[67,65],
[60,64],
[61,63] ]
def mirror_landmarks (landmarks, val):
result = landmarks.copy()
for idx in mirror_idxs:
result [ idx ] = result [ idx[::-1] ]
result[:,0] = val - result[:,0] - 1
return result
def get_face_struct_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,) ):
mask = np.zeros(image_shape[0:2]+( len(color),),dtype=np.float32)
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
draw_landmarks (mask, image_landmarks, color=color, draw_circles=False, thickness=2)
return mask
def draw_landmarks (image, image_landmarks, color=(0,255,0), draw_circles=True, thickness=1, transparent_mask=False):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
int_lmrks = np.array(image_landmarks, dtype=np.int)
jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])]
right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])]
left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])]
mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])]
right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])]
left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])]
nose = int_lmrks[slice(*landmarks_68_pt["nose"])]
# open shapes
cv2.polylines(image, tuple(np.array([v]) for v in ( right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])) )),
False, color, thickness=thickness, lineType=cv2.LINE_AA)
# closed shapes
cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)),
True, color, thickness=thickness, lineType=cv2.LINE_AA)
if draw_circles:
# the rest of the cicles
for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0):
cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA)
# jaw big circles
for x, y in jaw:
cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA)
if transparent_mask:
mask = get_image_hull_mask (image.shape, image_landmarks)
image[...] = ( image * (1-mask) + image * mask / 2 )[...]
def draw_rect_landmarks (image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False, landmarks_color=(0,255,0)):
draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask)
imagelib.draw_rect (image, rect, (255,0,0), 2 )
image_to_face_mat = get_transform_mat (image_landmarks, face_size, face_type)
points = transform_points ( [ (0,0), (0,face_size-1), (face_size-1, face_size-1), (face_size-1,0) ], image_to_face_mat, True)
imagelib.draw_polygon (image, points, (0,0,255), 2)
points = transform_points ( [ ( int(face_size*0.05), 0), ( int(face_size*0.1), int(face_size*0.1) ), ( 0, int(face_size*0.1) ) ], image_to_face_mat, True)
imagelib.draw_polygon (image, points, (0,0,255), 2)
def calc_face_pitch(landmarks):
if not isinstance(landmarks, np.ndarray):
landmarks = np.array (landmarks)
t = ( (landmarks[6][1]-landmarks[8][1]) + (landmarks[10][1]-landmarks[8][1]) ) / 2.0
b = landmarks[8][1]
return float(b-t)
def estimate_averaged_yaw(landmarks):
# Works much better than solvePnP if landmarks from "3DFAN"
if not isinstance(landmarks, np.ndarray):
landmarks = np.array (landmarks)
l = ( (landmarks[27][0]-landmarks[0][0]) + (landmarks[28][0]-landmarks[1][0]) + (landmarks[29][0]-landmarks[2][0]) ) / 3.0
r = ( (landmarks[16][0]-landmarks[27][0]) + (landmarks[15][0]-landmarks[28][0]) + (landmarks[14][0]-landmarks[29][0]) ) / 3.0
return float(r-l)
def estimate_pitch_yaw_roll(aligned_landmarks, size=256):
"""
returns pitch,yaw,roll [-pi/2...+pi/2]
"""
shape = (size,size)
focal_length = shape[1]
camera_center = (shape[1] / 2, shape[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, camera_center[0]],
[0, focal_length, camera_center[1]],
[0, 0, 1]], dtype=np.float32)
(_, rotation_vector, _) = cv2.solvePnP(
np.concatenate( (landmarks_68_3D[:27], landmarks_68_3D[30:36]) , axis=0) ,
np.concatenate( (aligned_landmarks[:27], aligned_landmarks[30:36]) , axis=0).astype(np.float32),
camera_matrix,
np.zeros((4, 1)) )
pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles( cv2.Rodrigues(rotation_vector)[0] )
half_pi = math.pi / 2.0
pitch = np.clip ( pitch, -half_pi, half_pi )
yaw = np.clip ( yaw , -half_pi, half_pi )
roll = np.clip ( roll, -half_pi, half_pi )
return -pitch, yaw, roll
#if remove_align:
# bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True)
# #import code
# #code.interact(local=dict(globals(), **locals()))
# area = mathlib.polygon_area(bbox[:,0], bbox[:,1] )
# side = math.sqrt(area) / 2
# center = transform_points ( [(output_size/2,output_size/2)], mat, True)
# pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] ))
# pts2 = np.float32([[0,0],[output_size,0],[0,output_size]])
# mat = cv2.getAffineTransform(pts1,pts2)
#if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN):
# #lmrks2 = expand_eyebrows(image_landmarks)
# #lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False )
# #y_diff = np.float32( (0,np.min(lmrks2_[:,1])) )
# #y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True)
# #y_diff = y_diff[1]-y_diff[0]
#
# x_diff = np.float32((0,0))
#
# lmrks2_ = transform_points( [ image_landmarks[0], image_landmarks[16] ], mat, False )
# if lmrks2_[0,0] < 0:
# x_diff = lmrks2_[0,0]
# x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
# x_diff = x_diff[1]-x_diff[0]
# elif lmrks2_[1,0] >= output_size:
# x_diff = lmrks2_[1,0]-(output_size-1)
# x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
# x_diff = x_diff[1]-x_diff[0]
#
# mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2)
"""
def get_averaged_transform_mat (img_landmarks,
img_landmarks_prev,
img_landmarks_next,
average_frame_count,
average_center_frame_count,
output_size, face_type, scale=1.0):
l_c_list = []
tb_diag_vec_list = []
bt_diag_vec_list = []
mod_list = []
count = max(average_frame_count,average_center_frame_count)
for i in range ( -count, count+1, 1 ):
if i < 0:
lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None
elif i > 0:
lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None
else:
lmrks = img_landmarks
if lmrks is None:
continue
l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale)
if i >= -average_frame_count and i <= average_frame_count:
tb_diag_vec_list.append(tb_diag_vec)
bt_diag_vec_list.append(bt_diag_vec)
mod_list.append(mod)
if i >= -average_center_frame_count and i <= average_center_frame_count:
l_c_list.append(l_c)
tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 )
bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 )
mod = np.mean( np.array(mod_list), axis=0 )
l_c = np.mean( np.array(l_c_list), axis=0 )
return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type)
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
# get face padding value for FaceType
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
# get corner points in global space
l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True)
l_c = l_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) )
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ) ] )
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ),
np.round( l_c - bt_diag_vec*mod ),
] )
# get area of face square in global space
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array( [ np.round( l_c + [-side,-side] ),
np.round( l_c + [ side,-side] ),
np.round( l_c + [ side, side] ) ] )
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
return mat
"""