diff --git a/decalib/datasets/datasets.py b/decalib/datasets/datasets.py index 61d87fdb..a2bcd541 100755 --- a/decalib/datasets/datasets.py +++ b/decalib/datasets/datasets.py @@ -46,7 +46,7 @@ def video2sequence(video_path, sample_step=10): return imagepath_list class TestData(Dataset): - def __init__(self, testpath, iscrop=True, crop_size=224, scale=1.25, face_detector='fan', sample_step=10): + def __init__(self, testpath, iscrop=True, crop_size=224, scale=1.25, face_detector='fan', sample_step=10, device='cuda'): ''' testpath: folder, imagepath_list, image path, video path ''' @@ -68,7 +68,7 @@ def __init__(self, testpath, iscrop=True, crop_size=224, scale=1.25, face_detect self.iscrop = iscrop self.resolution_inp = crop_size if face_detector == 'fan': - self.face_detector = detectors.FAN() + self.face_detector = detectors.FAN(device) # elif face_detector == 'mtcnn': # self.face_detector = detectors.MTCNN() else: diff --git a/decalib/datasets/detectors.py b/decalib/datasets/detectors.py index 631cd400..af22f1f6 100755 --- a/decalib/datasets/detectors.py +++ b/decalib/datasets/detectors.py @@ -17,9 +17,9 @@ import torch class FAN(object): - def __init__(self): + def __init__(self, device='cuda'): import face_alignment - self.model = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=False) + self.model = face_alignment.FaceAlignment(face_alignment.LandmarksType.TWO_D, flip_input=False, device=device) def run(self, image): ''' @@ -31,30 +31,31 @@ def run(self, image): return [0], 'kpt68' else: kpt = out[0].squeeze() - left = np.min(kpt[:,0]); right = np.max(kpt[:,0]); - top = np.min(kpt[:,1]); bottom = np.max(kpt[:,1]) - bbox = [left,top, right, bottom] + left = np.min(kpt[:, 0]); + right = np.max(kpt[:, 0]); + top = np.min(kpt[:, 1]); + bottom = np.max(kpt[:, 1]) + bbox = [left, top, right, bottom] return bbox, 'kpt68' + class MTCNN(object): - def __init__(self, device = 'cpu'): + def __init__(self, device='cpu'): ''' https://github.com/timesler/facenet-pytorch/blob/master/examples/infer.ipynb ''' from facenet_pytorch import MTCNN as mtcnn self.device = device self.model = mtcnn(keep_all=True) + def run(self, input): ''' image: 0-255, uint8, rgb, [h, w, 3] return: detected box ''' - out = self.model.detect(input[None,...]) + out = self.model.detect(input[None, ...]) if out[0][0] is None: return [0] else: bbox = out[0][0].squeeze() return bbox, 'bbox' - - - diff --git a/decalib/deca.py b/decalib/deca.py index bfa8bcdc..6dfe14c6 100644 --- a/decalib/deca.py +++ b/decalib/deca.py @@ -86,7 +86,10 @@ def _create_model(self, model_cfg): model_path = self.cfg.pretrained_modelpath if os.path.exists(model_path): print(f'trained model found. load {model_path}') - checkpoint = torch.load(model_path) + if self.device == 'cpu': + checkpoint = torch.load(model_path, map_location='cpu') + else: + checkpoint = torch.load(model_path) self.checkpoint = checkpoint util.copy_state_dict(self.E_flame.state_dict(), checkpoint['E_flame']) util.copy_state_dict(self.E_detail.state_dict(), checkpoint['E_detail']) diff --git a/decalib/utils/config.py b/decalib/utils/config.py index 9e306a00..5dcc38ec 100644 --- a/decalib/utils/config.py +++ b/decalib/utils/config.py @@ -1,10 +1,11 @@ ''' Default config for DECA ''' -from yacs.config import CfgNode as CN import argparse -import yaml import os +import yaml + +from yacs.config import CfgNode as CN cfg = CN() @@ -24,13 +25,13 @@ # texture data original from http://files.is.tue.mpg.de/tbolkart/FLAME/FLAME_texture_data.zip cfg.model.dense_template_path = os.path.join(cfg.deca_dir, 'data', 'texture_data_256.npy') cfg.model.fixed_displacement_path = os.path.join(cfg.deca_dir, 'data', 'fixed_displacement_256.npy') -cfg.model.flame_model_path = os.path.join(cfg.deca_dir, 'data', 'generic_model.pkl') -cfg.model.flame_lmk_embedding_path = os.path.join(cfg.deca_dir, 'data', 'landmark_embedding.npy') -cfg.model.face_mask_path = os.path.join(cfg.deca_dir, 'data', 'uv_face_mask.png') -cfg.model.face_eye_mask_path = os.path.join(cfg.deca_dir, 'data', 'uv_face_eye_mask.png') -cfg.model.mean_tex_path = os.path.join(cfg.deca_dir, 'data', 'mean_texture.jpg') -cfg.model.tex_path = os.path.join(cfg.deca_dir, 'data', 'FLAME_albedo_from_BFM.npz') -cfg.model.tex_type = 'BFM' # BFM, FLAME, albedoMM +cfg.model.flame_model_path = os.path.join(cfg.deca_dir, 'data', 'generic_model.pkl') +cfg.model.flame_lmk_embedding_path = os.path.join(cfg.deca_dir, 'data', 'landmark_embedding.npy') +cfg.model.face_mask_path = os.path.join(cfg.deca_dir, 'data', 'uv_face_mask.png') +cfg.model.face_eye_mask_path = os.path.join(cfg.deca_dir, 'data', 'uv_face_eye_mask.png') +cfg.model.mean_tex_path = os.path.join(cfg.deca_dir, 'data', 'mean_texture.jpg') +cfg.model.tex_path = os.path.join(cfg.deca_dir, 'data', 'FLAME_albedo_from_BFM.npz') +cfg.model.tex_type = 'BFM' # BFM, FLAME, albedoMM cfg.model.uv_size = 256 cfg.model.param_list = ['shape', 'tex', 'exp', 'pose', 'cam', 'light'] cfg.model.n_shape = 100 @@ -40,7 +41,8 @@ cfg.model.n_pose = 6 cfg.model.n_light = 27 cfg.model.use_tex = True -cfg.model.jaw_type = 'aa' # default use axis angle, another option: euler. Note that: aa is not stable in the beginning +cfg.model.extract_tex = True +cfg.model.jaw_type = 'aa' # default use axis angle, another option: euler. Note that: aa is not stable in the beginning # face recognition model cfg.model.fr_model_path = os.path.join(cfg.deca_dir, 'data', 'resnet50_ft_weight.pkl') @@ -100,7 +102,7 @@ cfg.loss.reg_exp = 1e-04 cfg.loss.reg_tex = 1e-04 cfg.loss.reg_light = 1. -cfg.loss.reg_jaw_pose = 0. #1. +cfg.loss.reg_jaw_pose = 0. # 1. cfg.loss.use_gender_prior = False cfg.loss.shape_consistency = True # loss for detail @@ -119,14 +121,16 @@ def get_cfg_defaults(): # This is for the "local variable" use pattern return cfg.clone() + def update_cfg(cfg, cfg_file): cfg.merge_from_file(cfg_file) return cfg.clone() + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--cfg', type=str, help='cfg file path') - parser.add_argument('--mode', type=str, default = 'train', help='deca mode') + parser.add_argument('--mode', type=str, default='train', help='deca mode') args = parser.parse_args() print(args, end='\n\n') diff --git a/decalib/utils/renderer.py b/decalib/utils/renderer.py index 6762c2b0..0f97db3e 100755 --- a/decalib/utils/renderer.py +++ b/decalib/utils/renderer.py @@ -19,9 +19,11 @@ import torch.nn.functional as F from skimage.io import imread import imageio + from . import util -def set_rasterizer(type = 'pytorch3d'): + +def set_rasterizer(type='standard'): if type == 'pytorch3d': global Meshes, load_obj, rasterize_meshes from pytorch3d.structures import Meshes @@ -31,18 +33,20 @@ def set_rasterizer(type = 'pytorch3d'): global standard_rasterize, load_obj import os from .util import load_obj - # Use JIT Compiling Extensions - # ref: https://pytorch.org/tutorials/advanced/cpp_extension.html + # Use JIT Compiling Extensions + # ref: https://pytorch.org/tutorials/advanced/cpp_extension.html from torch.utils.cpp_extension import load, CUDA_HOME curr_dir = os.path.dirname(__file__) standard_rasterize_cuda = \ - load(name='standard_rasterize_cuda', - sources=[f'{curr_dir}/rasterizer/standard_rasterize_cuda.cpp', f'{curr_dir}/rasterizer/standard_rasterize_cuda_kernel.cu'], - extra_cuda_cflags = ['-std=c++14', '-ccbin=$$(which gcc-7)']) # cuda10.2 is not compatible with gcc9. Specify gcc 7 + load(name='standard_rasterize_cuda', + sources=[f'{curr_dir}/rasterizer/standard_rasterize_cuda.cpp', + f'{curr_dir}/rasterizer/standard_rasterize_cuda_kernel.cu'], + extra_cuda_cflags=['-std=c++17', '-ccbin=$$(which gcc-11)']) from standard_rasterize_cuda import standard_rasterize - # If JIT does not work, try manually installation first - # 1. see instruction here: pixielib/utils/rasterizer/INSTALL.md - # 2. add this: "from .rasterizer.standard_rasterize_cuda import standard_rasterize" here + # If JIT does not work, try manually installation first + # 1. see instruction here: pixielib/utils/rasterizer/INSTALL.md + # 2. add this: "from .rasterizer.standard_rasterize_cuda import standard_rasterize" here + class StandardRasterizer(nn.Module): """ Alg: https://www.scratchapixel.com/lessons/3d-basic-rendering/rasterization-practical-implementation @@ -51,6 +55,7 @@ class StandardRasterizer(nn.Module): can render non-squared image not differentiable """ + def __init__(self, height, width=None): """ use fixed raster_settings for rendering faces @@ -58,14 +63,15 @@ def __init__(self, height, width=None): super().__init__() if width is None: width = height - self.h = h = height; self.w = w = width + self.h = h = height; + self.w = w = width def forward(self, vertices, faces, attributes=None, h=None, w=None): device = vertices.device if h is None: h = self.h if w is None: - w = self.h; + w = self.h; bz = vertices.shape[0] depth_buffer = torch.zeros([bz, h, w]).float().to(device) + 1e6 triangle_buffer = torch.zeros([bz, h, w]).int().to(device) - 1 @@ -73,26 +79,27 @@ def forward(self, vertices, faces, attributes=None, h=None, w=None): vert_vis = torch.zeros([bz, vertices.shape[1]]).float().to(device) vertices = vertices.clone().float() # compatibale with pytorch3d ndc, see https://github.com/facebookresearch/pytorch3d/blob/e42b0c4f704fa0f5e262f370dccac537b5edf2b1/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.cu#L232 - vertices[...,:2] = -vertices[...,:2] - vertices[...,0] = vertices[..., 0]*w/2 + w/2 - vertices[...,1] = vertices[..., 1]*h/2 + h/2 - vertices[...,0] = w - 1 - vertices[..., 0] - vertices[...,1] = h - 1 - vertices[..., 1] - vertices[...,0] = -1 + (2*vertices[...,0] + 1)/w - vertices[...,1] = -1 + (2*vertices[...,1] + 1)/h + vertices[..., :2] = -vertices[..., :2] + vertices[..., 0] = vertices[..., 0] * w / 2 + w / 2 + vertices[..., 1] = vertices[..., 1] * h / 2 + h / 2 + vertices[..., 0] = w - 1 - vertices[..., 0] + vertices[..., 1] = h - 1 - vertices[..., 1] + vertices[..., 0] = -1 + (2 * vertices[..., 0] + 1) / w + vertices[..., 1] = -1 + (2 * vertices[..., 1] + 1) / h # vertices = vertices.clone().float() - vertices[...,0] = vertices[..., 0]*w/2 + w/2 - vertices[...,1] = vertices[..., 1]*h/2 + h/2 - vertices[...,2] = vertices[..., 2]*w/2 + vertices[..., 0] = vertices[..., 0] * w / 2 + w / 2 + vertices[..., 1] = vertices[..., 1] * h / 2 + h / 2 + vertices[..., 2] = vertices[..., 2] * w / 2 f_vs = util.face_vertices(vertices, faces) standard_rasterize(f_vs, depth_buffer, triangle_buffer, baryw_buffer, h, w) - pix_to_face = triangle_buffer[:,:,:,None].long() - bary_coords = baryw_buffer[:,:,:,None,:] + pix_to_face = triangle_buffer[:, :, :, None].long() + bary_coords = baryw_buffer[:, :, :, None, :] vismask = (pix_to_face > -1).float() D = attributes.shape[-1] - attributes = attributes.clone(); attributes = attributes.view(attributes.shape[0]*attributes.shape[1], 3, attributes.shape[-1]) + attributes = attributes.clone(); + attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) N, H, W, K, _ = bary_coords.shape mask = pix_to_face == -1 pix_to_face = pix_to_face.clone() @@ -101,10 +108,11 @@ def forward(self, vertices, faces, attributes=None, h=None, w=None): pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (bary_coords[..., None] * pixel_face_vals).sum(dim=-2) pixel_vals[mask] = 0 # Replace masked values in output. - pixel_vals = pixel_vals[:,:,:,0].permute(0,3,1,2) - pixel_vals = torch.cat([pixel_vals, vismask[:,:,:,0][:,None,:,:]], dim=1) + pixel_vals = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) + pixel_vals = torch.cat([pixel_vals, vismask[:, :, :, 0][:, None, :, :]], dim=1) return pixel_vals + class Pytorch3dRasterizer(nn.Module): ## TODO: add support for rendering non-squared images, since pytorc3d supports this now """ Borrowed from https://github.com/facebookresearch/pytorch3d @@ -123,7 +131,7 @@ def __init__(self, image_size=224): 'blur_radius': 0.0, 'faces_per_pixel': 1, 'bin_size': None, - 'max_faces_per_bin': None, + 'max_faces_per_bin': None, 'perspective_correct': False, } raster_settings = util.dict2obj(raster_settings) @@ -131,17 +139,17 @@ def __init__(self, image_size=224): def forward(self, vertices, faces, attributes=None, h=None, w=None): fixed_vertices = vertices.clone() - fixed_vertices[...,:2] = -fixed_vertices[...,:2] + fixed_vertices[..., :2] = -fixed_vertices[..., :2] raster_settings = self.raster_settings if h is None and w is None: image_size = raster_settings.image_size else: image_size = [h, w] - if h>w: - fixed_vertices[..., 1] = fixed_vertices[..., 1]*h/w + if h > w: + fixed_vertices[..., 1] = fixed_vertices[..., 1] * h / w else: - fixed_vertices[..., 0] = fixed_vertices[..., 0]*w/h - + fixed_vertices[..., 0] = fixed_vertices[..., 0] * w / h + meshes_screen = Meshes(verts=fixed_vertices.float(), faces=faces.long()) pix_to_face, zbuf, bary_coords, dists = rasterize_meshes( meshes_screen, @@ -154,7 +162,8 @@ def forward(self, vertices, faces, attributes=None, h=None, w=None): ) vismask = (pix_to_face > -1).float() D = attributes.shape[-1] - attributes = attributes.clone(); attributes = attributes.view(attributes.shape[0]*attributes.shape[1], 3, attributes.shape[-1]) + attributes = attributes.clone(); + attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) N, H, W, K, _ = bary_coords.shape mask = pix_to_face == -1 pix_to_face = pix_to_face.clone() @@ -163,12 +172,13 @@ def forward(self, vertices, faces, attributes=None, h=None, w=None): pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (bary_coords[..., None] * pixel_face_vals).sum(dim=-2) pixel_vals[mask] = 0 # Replace masked values in output. - pixel_vals = pixel_vals[:,:,:,0].permute(0,3,1,2) - pixel_vals = torch.cat([pixel_vals, vismask[:,:,:,0][:,None,:,:]], dim=1) + pixel_vals = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) + pixel_vals = torch.cat([pixel_vals, vismask[:, :, :, 0][:, None, :, :]], dim=1) # print(image_size) # import ipdb; ipdb.set_trace() return pixel_vals + class SRenderY(nn.Module): def __init__(self, image_size, obj_filename, uv_size=256, rasterizer_type='pytorch3d'): super(SRenderY, self).__init__() @@ -178,9 +188,9 @@ def __init__(self, image_size, obj_filename, uv_size=256, rasterizer_type='pytor self.rasterizer = Pytorch3dRasterizer(image_size) self.uv_rasterizer = Pytorch3dRasterizer(uv_size) verts, faces, aux = load_obj(obj_filename) - uvcoords = aux.verts_uvs[None, ...] # (N, V, 2) - uvfaces = faces.textures_idx[None, ...] # (N, F, 3) - faces = faces.verts_idx[None,...] + uvcoords = aux.verts_uvs[None, ...] # (N, V, 2) + uvfaces = faces.textures_idx[None, ...] # (N, F, 3) + faces = faces.verts_idx[None, ...] elif rasterizer_type == 'standard': self.rasterizer = StandardRasterizer(image_size) self.uv_rasterizer = StandardRasterizer(uv_size) @@ -194,31 +204,36 @@ def __init__(self, image_size, obj_filename, uv_size=256, rasterizer_type='pytor # faces dense_triangles = util.generate_triangles(uv_size, uv_size) - self.register_buffer('dense_faces', torch.from_numpy(dense_triangles).long()[None,:,:]) + self.register_buffer('dense_faces', torch.from_numpy(dense_triangles).long()[None, :, :]) self.register_buffer('faces', faces) self.register_buffer('raw_uvcoords', uvcoords) # uv coords - uvcoords = torch.cat([uvcoords, uvcoords[:,:,0:1]*0.+1.], -1) #[bz, ntv, 3] - uvcoords = uvcoords*2 - 1; uvcoords[...,1] = -uvcoords[...,1] + uvcoords = torch.cat([uvcoords, uvcoords[:, :, 0:1] * 0. + 1.], -1) # [bz, ntv, 3] + uvcoords = uvcoords * 2 - 1; + uvcoords[..., 1] = -uvcoords[..., 1] face_uvcoords = util.face_vertices(uvcoords, uvfaces) self.register_buffer('uvcoords', uvcoords) self.register_buffer('uvfaces', uvfaces) self.register_buffer('face_uvcoords', face_uvcoords) # shape colors, for rendering shape overlay - colors = torch.tensor([180, 180, 180])[None, None, :].repeat(1, faces.max()+1, 1).float()/255. + colors = torch.tensor([180, 180, 180])[None, None, :].repeat(1, faces.max() + 1, 1).float() / 255. face_colors = util.face_vertices(colors, faces) self.register_buffer('face_colors', face_colors) ## SH factors for lighting pi = np.pi - constant_factor = torch.tensor([1/np.sqrt(4*pi), ((2*pi)/3)*(np.sqrt(3/(4*pi))), ((2*pi)/3)*(np.sqrt(3/(4*pi))),\ - ((2*pi)/3)*(np.sqrt(3/(4*pi))), (pi/4)*(3)*(np.sqrt(5/(12*pi))), (pi/4)*(3)*(np.sqrt(5/(12*pi))),\ - (pi/4)*(3)*(np.sqrt(5/(12*pi))), (pi/4)*(3/2)*(np.sqrt(5/(12*pi))), (pi/4)*(1/2)*(np.sqrt(5/(4*pi)))]).float() + constant_factor = torch.tensor( + [1 / np.sqrt(4 * pi), ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))), ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))), \ + ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))), (pi / 4) * (3) * (np.sqrt(5 / (12 * pi))), + (pi / 4) * (3) * (np.sqrt(5 / (12 * pi))), \ + (pi / 4) * (3) * (np.sqrt(5 / (12 * pi))), (pi / 4) * (3 / 2) * (np.sqrt(5 / (12 * pi))), + (pi / 4) * (1 / 2) * (np.sqrt(5 / (4 * pi)))]).float() self.register_buffer('constant_factor', constant_factor) - - def forward(self, vertices, transformed_vertices, albedos, lights=None, h=None, w=None, light_type='point', background=None): + + def forward(self, vertices, transformed_vertices, albedos, lights=None, h=None, w=None, light_type='point', + background=None): ''' -- Texture Rendering vertices: [batch_size, V, 3], vertices in world space, for calculating normals, then shading @@ -232,26 +247,29 @@ def forward(self, vertices, transformed_vertices, albedos, lights=None, h=None, ''' batch_size = vertices.shape[0] ## rasterizer near 0 far 100. move mesh so minz larger than 0 - transformed_vertices[:,:,2] = transformed_vertices[:,:,2] + 10 + transformed_vertices[:, :, 2] = transformed_vertices[:, :, 2] + 10 # attributes face_vertices = util.face_vertices(vertices, self.faces.expand(batch_size, -1, -1)) - normals = util.vertex_normals(vertices, self.faces.expand(batch_size, -1, -1)); face_normals = util.face_vertices(normals, self.faces.expand(batch_size, -1, -1)) - transformed_normals = util.vertex_normals(transformed_vertices, self.faces.expand(batch_size, -1, -1)); transformed_face_normals = util.face_vertices(transformed_normals, self.faces.expand(batch_size, -1, -1)) - - attributes = torch.cat([self.face_uvcoords.expand(batch_size, -1, -1, -1), - transformed_face_normals.detach(), - face_vertices.detach(), - face_normals], - -1) + normals = util.vertex_normals(vertices, self.faces.expand(batch_size, -1, -1)); + face_normals = util.face_vertices(normals, self.faces.expand(batch_size, -1, -1)) + transformed_normals = util.vertex_normals(transformed_vertices, self.faces.expand(batch_size, -1, -1)); + transformed_face_normals = util.face_vertices(transformed_normals, self.faces.expand(batch_size, -1, -1)) + + attributes = torch.cat([self.face_uvcoords.expand(batch_size, -1, -1, -1), + transformed_face_normals.detach(), + face_vertices.detach(), + face_normals], + -1) # rasterize rendering = self.rasterizer(transformed_vertices, self.faces.expand(batch_size, -1, -1), attributes, h, w) - + #### # vis mask alpha_images = rendering[:, -1, :, :][:, None, :, :].detach() # albedo - uvcoords_images = rendering[:, :3, :, :]; grid = (uvcoords_images).permute(0, 2, 3, 1)[:, :, :, :2] + uvcoords_images = rendering[:, :3, :, :]; + grid = (uvcoords_images).permute(0, 2, 3, 1)[:, :, :, :2] albedo_images = F.grid_sample(albedos, grid, align_corners=False) # visible mask for pixels with positive normal direction @@ -264,24 +282,29 @@ def forward(self, vertices, transformed_vertices, albedos, lights=None, h=None, if lights.shape[1] == 9: shading_images = self.add_SHlight(normal_images, lights) else: - if light_type=='point': + if light_type == 'point': vertice_images = rendering[:, 6:9, :, :].detach() - shading = self.add_pointlight(vertice_images.permute(0,2,3,1).reshape([batch_size, -1, 3]), normal_images.permute(0,2,3,1).reshape([batch_size, -1, 3]), lights) - shading_images = shading.reshape([batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0,3,1,2) + shading = self.add_pointlight(vertice_images.permute(0, 2, 3, 1).reshape([batch_size, -1, 3]), + normal_images.permute(0, 2, 3, 1).reshape([batch_size, -1, 3]), + lights) + shading_images = shading.reshape( + [batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0, 3, 1, 2) else: - shading = self.add_directionlight(normal_images.permute(0,2,3,1).reshape([batch_size, -1, 3]), lights) - shading_images = shading.reshape([batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0,3,1,2) - images = albedo_images*shading_images + shading = self.add_directionlight(normal_images.permute(0, 2, 3, 1).reshape([batch_size, -1, 3]), + lights) + shading_images = shading.reshape( + [batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0, 3, 1, 2) + images = albedo_images * shading_images else: images = albedo_images - shading_images = images.detach()*0. + shading_images = images.detach() * 0. if background is not None: - images = images*alpha_images + background*(1.-alpha_images) - albedo_images = albedo_images*alpha_images + background*(1.-alpha_images) + images = images * alpha_images + background * (1. - alpha_images) + albedo_images = albedo_images * alpha_images + background * (1. - alpha_images) else: - images = images*alpha_images - albedo_images = albedo_images*alpha_images + images = images * alpha_images + albedo_images = albedo_images * alpha_images outputs = { 'images': images, @@ -291,10 +314,10 @@ def forward(self, vertices, transformed_vertices, albedos, lights=None, h=None, 'shading_images': shading_images, 'grid': grid, 'normals': normals, - 'normal_images': normal_images*alpha_images, + 'normal_images': normal_images * alpha_images, 'transformed_normals': transformed_normals, } - + return outputs def add_SHlight(self, normal_images, sh_coeff): @@ -303,13 +326,13 @@ def add_SHlight(self, normal_images, sh_coeff): ''' N = normal_images sh = torch.stack([ - N[:,0]*0.+1., N[:,0], N[:,1], \ - N[:,2], N[:,0]*N[:,1], N[:,0]*N[:,2], - N[:,1]*N[:,2], N[:,0]**2 - N[:,1]**2, 3*(N[:,2]**2) - 1 - ], - 1) # [bz, 9, h, w] - sh = sh*self.constant_factor[None,:,None,None] - shading = torch.sum(sh_coeff[:,:,:,None,None]*sh[:,:,None,:,:], 1) # [bz, 9, 3, h, w] + N[:, 0] * 0. + 1., N[:, 0], N[:, 1], \ + N[:, 2], N[:, 0] * N[:, 1], N[:, 0] * N[:, 2], + N[:, 1] * N[:, 2], N[:, 0] ** 2 - N[:, 1] ** 2, 3 * (N[:, 2] ** 2) - 1 + ], + 1) # [bz, 9, h, w] + sh = sh * self.constant_factor[None, :, None, None] + shading = torch.sum(sh_coeff[:, :, :, None, None] * sh[:, :, None, :, :], 1) # [bz, 9, 3, h, w] return shading def add_pointlight(self, vertices, normals, lights): @@ -319,11 +342,12 @@ def add_pointlight(self, vertices, normals, lights): returns: shading: [bz, nv, 3] ''' - light_positions = lights[:,:,:3]; light_intensities = lights[:,:,3:] - directions_to_lights = F.normalize(light_positions[:,:,None,:] - vertices[:,None,:,:], dim=3) + light_positions = lights[:, :, :3]; + light_intensities = lights[:, :, 3:] + directions_to_lights = F.normalize(light_positions[:, :, None, :] - vertices[:, None, :, :], dim=3) # normals_dot_lights = torch.clamp((normals[:,None,:,:]*directions_to_lights).sum(dim=3), 0., 1.) - normals_dot_lights = (normals[:,None,:,:]*directions_to_lights).sum(dim=3) - shading = normals_dot_lights[:,:,:,None]*light_intensities[:,:,None,:] + normals_dot_lights = (normals[:, None, :, :] * directions_to_lights).sum(dim=3) + shading = normals_dot_lights[:, :, :, None] * light_intensities[:, :, None, :] return shading.mean(1) def add_directionlight(self, normals, lights): @@ -333,16 +357,17 @@ def add_directionlight(self, normals, lights): returns: shading: [bz, nv, 3] ''' - light_direction = lights[:,:,:3]; light_intensities = lights[:,:,3:] - directions_to_lights = F.normalize(light_direction[:,:,None,:].expand(-1,-1,normals.shape[1],-1), dim=3) + light_direction = lights[:, :, :3]; + light_intensities = lights[:, :, 3:] + directions_to_lights = F.normalize(light_direction[:, :, None, :].expand(-1, -1, normals.shape[1], -1), dim=3) # normals_dot_lights = torch.clamp((normals[:,None,:,:]*directions_to_lights).sum(dim=3), 0., 1.) # normals_dot_lights = (normals[:,None,:,:]*directions_to_lights).sum(dim=3) - normals_dot_lights = torch.clamp((normals[:,None,:,:]*directions_to_lights).sum(dim=3), 0., 1.) - shading = normals_dot_lights[:,:,:,None]*light_intensities[:,:,None,:] + normals_dot_lights = torch.clamp((normals[:, None, :, :] * directions_to_lights).sum(dim=3), 0., 1.) + shading = normals_dot_lights[:, :, :, None] * light_intensities[:, :, None, :] return shading.mean(1) - def render_shape(self, vertices, transformed_vertices, colors = None, images=None, detail_normal_images=None, - lights=None, return_grid=False, uv_detail_normals=None, h=None, w=None): + def render_shape(self, vertices, transformed_vertices, colors=None, images=None, detail_normal_images=None, + lights=None, return_grid=False, uv_detail_normals=None, h=None, w=None): ''' -- rendering shape with detail normal map ''' @@ -351,29 +376,31 @@ def render_shape(self, vertices, transformed_vertices, colors = None, images=Non if lights is None: light_positions = torch.tensor( [ - [-1,1,1], - [1,1,1], - [-1,-1,1], - [1,-1,1], - [0,0,1] + [-1, 1, 1], + [1, 1, 1], + [-1, -1, 1], + [1, -1, 1], + [0, 0, 1] ] - )[None,:,:].expand(batch_size, -1, -1).float() - light_intensities = torch.ones_like(light_positions).float()*1.7 + )[None, :, :].expand(batch_size, -1, -1).float() + light_intensities = torch.ones_like(light_positions).float() * 1.7 lights = torch.cat((light_positions, light_intensities), 2).to(vertices.device) - transformed_vertices[:,:,2] = transformed_vertices[:,:,2] + 10 + transformed_vertices[:, :, 2] = transformed_vertices[:, :, 2] + 10 # Attributes face_vertices = util.face_vertices(vertices, self.faces.expand(batch_size, -1, -1)) - normals = util.vertex_normals(vertices, self.faces.expand(batch_size, -1, -1)); face_normals = util.face_vertices(normals, self.faces.expand(batch_size, -1, -1)) - transformed_normals = util.vertex_normals(transformed_vertices, self.faces.expand(batch_size, -1, -1)); transformed_face_normals = util.face_vertices(transformed_normals, self.faces.expand(batch_size, -1, -1)) + normals = util.vertex_normals(vertices, self.faces.expand(batch_size, -1, -1)); + face_normals = util.face_vertices(normals, self.faces.expand(batch_size, -1, -1)) + transformed_normals = util.vertex_normals(transformed_vertices, self.faces.expand(batch_size, -1, -1)); + transformed_face_normals = util.face_vertices(transformed_normals, self.faces.expand(batch_size, -1, -1)) if colors is None: colors = self.face_colors.expand(batch_size, -1, -1, -1) - attributes = torch.cat([colors, - transformed_face_normals.detach(), - face_vertices.detach(), - face_normals, - self.face_uvcoords.expand(batch_size, -1, -1, -1)], - -1) + attributes = torch.cat([colors, + transformed_face_normals.detach(), + face_vertices.detach(), + face_normals, + self.face_uvcoords.expand(batch_size, -1, -1, -1)], + -1) # rasterize # import ipdb; ipdb.set_trace() rendering = self.rasterizer(transformed_vertices, self.faces.expand(batch_size, -1, -1), attributes, h, w) @@ -393,43 +420,46 @@ def render_shape(self, vertices, transformed_vertices, colors = None, images=Non if detail_normal_images is not None: normal_images = detail_normal_images - shading = self.add_directionlight(normal_images.permute(0,2,3,1).reshape([batch_size, -1, 3]), lights) - shading_images = shading.reshape([batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0,3,1,2).contiguous() - shaded_images = albedo_images*shading_images + shading = self.add_directionlight(normal_images.permute(0, 2, 3, 1).reshape([batch_size, -1, 3]), lights) + shading_images = shading.reshape([batch_size, albedo_images.shape[2], albedo_images.shape[3], 3]).permute(0, 3, + 1, + 2).contiguous() + shaded_images = albedo_images * shading_images - alpha_images = alpha_images*pos_mask + alpha_images = alpha_images * pos_mask if images is None: - shape_images = shaded_images*alpha_images + torch.zeros_like(shaded_images).to(vertices.device)*(1-alpha_images) + shape_images = shaded_images * alpha_images + torch.zeros_like(shaded_images).to(vertices.device) * ( + 1 - alpha_images) else: - shape_images = shaded_images*alpha_images + images*(1-alpha_images) + shape_images = shaded_images * alpha_images + images * (1 - alpha_images) if return_grid: - uvcoords_images = rendering[:, 12:15, :, :]; + uvcoords_images = rendering[:, 12:15, :, :]; grid = (uvcoords_images).permute(0, 2, 3, 1)[:, :, :, :2] return shape_images, normal_images, grid, alpha_images else: return shape_images - + def render_depth(self, transformed_vertices): ''' -- rendering depth ''' batch_size = transformed_vertices.shape[0] - transformed_vertices[:,:,2] = transformed_vertices[:,:,2] - transformed_vertices[:,:,2].min() - z = -transformed_vertices[:,:,2:].repeat(1,1,3).clone() - z = z-z.min() - z = z/z.max() + transformed_vertices[:, :, 2] = transformed_vertices[:, :, 2] - transformed_vertices[:, :, 2].min() + z = -transformed_vertices[:, :, 2:].repeat(1, 1, 3).clone() + z = z - z.min() + z = z / z.max() # Attributes attributes = util.face_vertices(z, self.faces.expand(batch_size, -1, -1)) # rasterize - transformed_vertices[:,:,2] = transformed_vertices[:,:,2] + 10 + transformed_vertices[:, :, 2] = transformed_vertices[:, :, 2] + 10 rendering = self.rasterizer(transformed_vertices, self.faces.expand(batch_size, -1, -1), attributes) #### alpha_images = rendering[:, -1, :, :][:, None, :, :].detach() depth_images = rendering[:, :1, :, :] return depth_images - + def render_colors(self, transformed_vertices, colors): ''' -- rendering colors: could be rgb color/ normals, etc @@ -443,7 +473,7 @@ def render_colors(self, transformed_vertices, colors): rendering = self.rasterizer(transformed_vertices, self.faces.expand(batch_size, -1, -1), attributes) #### alpha_images = rendering[:, [-1], :, :].detach() - images = rendering[:, :3, :, :]* alpha_images + images = rendering[:, :3, :, :] * alpha_images return images def world2uv(self, vertices): @@ -454,5 +484,6 @@ def world2uv(self, vertices): ''' batch_size = vertices.shape[0] face_vertices = util.face_vertices(vertices, self.faces.expand(batch_size, -1, -1)) - uv_vertices = self.uv_rasterizer(self.uvcoords.expand(batch_size, -1, -1), self.uvfaces.expand(batch_size, -1, -1), face_vertices)[:, :3] - return uv_vertices \ No newline at end of file + uv_vertices = self.uv_rasterizer(self.uvcoords.expand(batch_size, -1, -1), + self.uvfaces.expand(batch_size, -1, -1), face_vertices)[:, :3] + return uv_vertices diff --git a/demos/demo_reconstruct.py b/demos/demo_reconstruct.py index 12eec79e..3162393d 100755 --- a/demos/demo_reconstruct.py +++ b/demos/demo_reconstruct.py @@ -22,9 +22,10 @@ from tqdm import tqdm import torch + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from decalib.deca import DECA -from decalib.datasets import datasets +from decalib.datasets import datasets from decalib.utils import util from decalib.utils.config import cfg as deca_cfg from decalib.utils.tensor_cropper import transform_points @@ -37,32 +38,33 @@ def main(args): os.makedirs(savefolder, exist_ok=True) # load test images - testdata = datasets.TestData(args.inputpath, iscrop=args.iscrop, face_detector=args.detector, sample_step=args.sample_step) + testdata = datasets.TestData(args.inputpath, iscrop=args.iscrop, face_detector=args.detector, + sample_step=args.sample_step, device=args.device) # run DECA deca_cfg.model.use_tex = args.useTex deca_cfg.rasterizer_type = args.rasterizer_type deca_cfg.model.extract_tex = args.extractTex - deca = DECA(config = deca_cfg, device=device) + deca = DECA(config=deca_cfg, device=device) # for i in range(len(testdata)): for i in tqdm(range(len(testdata))): name = testdata[i]['imagename'] - images = testdata[i]['image'].to(device)[None,...] + images = testdata[i]['image'].to(device)[None, ...] with torch.no_grad(): codedict = deca.encode(images) - opdict, visdict = deca.decode(codedict) #tensor + opdict, visdict = deca.decode(codedict) # tensor if args.render_orig: tform = testdata[i]['tform'][None, ...] - tform = torch.inverse(tform).transpose(1,2).to(device) + tform = torch.inverse(tform).transpose(1, 2).to(device) original_image = testdata[i]['original_image'][None, ...].to(device) - _, orig_visdict = deca.decode(codedict, render_orig=True, original_image=original_image, tform=tform) - orig_visdict['inputs'] = original_image + _, orig_visdict = deca.decode(codedict, render_orig=True, original_image=original_image, tform=tform) + orig_visdict['inputs'] = original_image if args.saveDepth or args.saveKpt or args.saveObj or args.saveMat or args.saveImages: os.makedirs(os.path.join(savefolder, name), exist_ok=True) # -- save results if args.saveDepth: - depth_image = deca.render.render_depth(opdict['trans_verts']).repeat(1,3,1,1) + depth_image = deca.render.render_depth(opdict['trans_verts']).repeat(1, 3, 1, 1) visdict['depth_images'] = depth_image cv2.imwrite(os.path.join(savefolder, name, name + '_depth.jpg'), util.tensor2image(depth_image[0])) if args.saveKpt: @@ -78,16 +80,20 @@ def main(args): if args.render_orig: cv2.imwrite(os.path.join(savefolder, name + '_vis_original_size.jpg'), deca.visualize(orig_visdict)) if args.saveImages: - for vis_name in ['inputs', 'rendered_images', 'albedo_images', 'shape_images', 'shape_detail_images', 'landmarks2d']: + for vis_name in ['inputs', 'rendered_images', 'albedo_images', 'shape_images', 'shape_detail_images', + 'landmarks2d']: if vis_name not in visdict.keys(): continue image = util.tensor2image(visdict[vis_name][0]) - cv2.imwrite(os.path.join(savefolder, name, name + '_' + vis_name +'.jpg'), util.tensor2image(visdict[vis_name][0])) + cv2.imwrite(os.path.join(savefolder, name, name + '_' + vis_name + '.jpg'), + util.tensor2image(visdict[vis_name][0])) if args.render_orig: image = util.tensor2image(orig_visdict[vis_name][0]) - cv2.imwrite(os.path.join(savefolder, name, 'orig_' + name + '_' + vis_name +'.jpg'), util.tensor2image(orig_visdict[vis_name][0])) + cv2.imwrite(os.path.join(savefolder, name, 'orig_' + name + '_' + vis_name + '.jpg'), + util.tensor2image(orig_visdict[vis_name][0])) print(f'-- please check the results in {savefolder}') - + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='DECA: Detailed Expression Capture and Animation') @@ -96,36 +102,36 @@ def main(args): parser.add_argument('-s', '--savefolder', default='TestSamples/examples/results', type=str, help='path to the output directory, where results(obj, txt files) will be stored.') parser.add_argument('--device', default='cuda', type=str, - help='set device, cpu for using cpu' ) + help='set device, cpu for using cpu') # process test images parser.add_argument('--iscrop', default=True, type=lambda x: x.lower() in ['true', '1'], - help='whether to crop input image, set false only when the test image are well cropped' ) + help='whether to crop input image, set false only when the test image are well cropped') parser.add_argument('--sample_step', default=10, type=int, - help='sample images from video data for every step' ) + help='sample images from video data for every step') parser.add_argument('--detector', default='fan', type=str, - help='detector for cropping face, check decalib/detectors.py for details' ) + help='detector for cropping face, check decalib/detectors.py for details') # rendering option parser.add_argument('--rasterizer_type', default='standard', type=str, - help='rasterizer type: pytorch3d or standard' ) + help='rasterizer type: pytorch3d or standard') parser.add_argument('--render_orig', default=True, type=lambda x: x.lower() in ['true', '1'], help='whether to render results in original image size, currently only works when rasterizer_type=standard') # save parser.add_argument('--useTex', default=False, type=lambda x: x.lower() in ['true', '1'], help='whether to use FLAME texture model to generate uv texture map, \ - set it to True only if you downloaded texture model' ) + set it to True only if you downloaded texture model') parser.add_argument('--extractTex', default=True, type=lambda x: x.lower() in ['true', '1'], - help='whether to extract texture from input image as the uv texture map, set false if you want albeo map from FLAME mode' ) + help='whether to extract texture from input image as the uv texture map, set false if you want albeo map from FLAME mode') parser.add_argument('--saveVis', default=True, type=lambda x: x.lower() in ['true', '1'], - help='whether to save visualization of output' ) + help='whether to save visualization of output') parser.add_argument('--saveKpt', default=False, type=lambda x: x.lower() in ['true', '1'], - help='whether to save 2D and 3D keypoints' ) + help='whether to save 2D and 3D keypoints') parser.add_argument('--saveDepth', default=False, type=lambda x: x.lower() in ['true', '1'], - help='whether to save depth image' ) + help='whether to save depth image') parser.add_argument('--saveObj', default=False, type=lambda x: x.lower() in ['true', '1'], help='whether to save outputs as .obj, detail mesh will end with _detail.obj. \ - Note that saving objs could be slow' ) + Note that saving objs could be slow') parser.add_argument('--saveMat', default=False, type=lambda x: x.lower() in ['true', '1'], - help='whether to save outputs as .mat' ) + help='whether to save outputs as .mat') parser.add_argument('--saveImages', default=False, type=lambda x: x.lower() in ['true', '1'], - help='whether to save visualization output as seperate images' ) - main(parser.parse_args()) \ No newline at end of file + help='whether to save visualization output as seperate images') + main(parser.parse_args()) diff --git a/demos/demo_teaser.py b/demos/demo_teaser.py index 71f480f9..ee5b0413 100755 --- a/demos/demo_teaser.py +++ b/demos/demo_teaser.py @@ -25,19 +25,20 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from decalib.deca import DECA -from decalib.datasets import datasets +from decalib.datasets import datasets from decalib.utils import util from decalib.utils.rotation_converter import batch_euler2axis, deg2rad from decalib.utils.config import cfg as deca_cfg + def main(args): savefolder = args.savefolder device = args.device os.makedirs(savefolder, exist_ok=True) # load test images - testdata = datasets.TestData(args.inputpath, iscrop=args.iscrop, face_detector=args.detector) - expdata = datasets.TestData(args.exp_path, iscrop=args.iscrop, face_detector=args.detector) + testdata = datasets.TestData(args.inputpath, iscrop=args.iscrop, face_detector=args.detector, device=args.device) + expdata = datasets.TestData(args.exp_path, iscrop=args.iscrop, face_detector=args.detector, device=args.device) # DECA deca_cfg.rasterizer_type = args.rasterizer_type deca = DECA(config=deca_cfg, device=device) @@ -45,56 +46,56 @@ def main(args): visdict_list_list = [] for i in range(len(testdata)): name = testdata[i]['imagename'] - images = testdata[i]['image'].to(device)[None,...] + images = testdata[i]['image'].to(device)[None, ...] with torch.no_grad(): codedict = deca.encode(images) - opdict, visdict = deca.decode(codedict) #tensor + opdict, visdict = deca.decode(codedict) # tensor ### show shape with different views and expressions visdict_list = [] max_yaw = 30 - yaw_list = list(range(0,max_yaw,5)) + list(range(max_yaw,-max_yaw,-5)) + list(range(-max_yaw,0,5)) - for k in yaw_list: #jaw angle from -50 to 50 + yaw_list = list(range(0, max_yaw, 5)) + list(range(max_yaw, -max_yaw, -5)) + list(range(-max_yaw, 0, 5)) + for k in yaw_list: # jaw angle from -50 to 50 ## yaw angle euler_pose = torch.randn((1, 3)) - euler_pose[:,1] = k#torch.rand((self.batch_size))*160 - 80 - euler_pose[:,0] = 0#(torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) - euler_pose[:,2] = 0#(torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) - global_pose = batch_euler2axis(deg2rad(euler_pose[:,:3].cuda())) - codedict['pose'][:,:3] = global_pose - codedict['cam'][:,:] = 0. - codedict['cam'][:,0] = 8 - _, visdict_view = deca.decode(codedict) - visdict = {x:visdict[x] for x in ['inputs', 'shape_detail_images']} + euler_pose[:, 1] = k # torch.rand((self.batch_size))*160 - 80 + euler_pose[:, 0] = 0 # (torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) + euler_pose[:, 2] = 0 # (torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) + global_pose = batch_euler2axis(deg2rad(euler_pose[:, :3].cuda())) + codedict['pose'][:, :3] = global_pose + codedict['cam'][:, :] = 0. + codedict['cam'][:, 0] = 8 + _, visdict_view = deca.decode(codedict) + visdict = {x: visdict[x] for x in ['inputs', 'shape_detail_images']} visdict['pose'] = visdict_view['shape_detail_images'] visdict_list.append(visdict) euler_pose = torch.zeros((1, 3)) - global_pose = batch_euler2axis(deg2rad(euler_pose[:,:3].cuda())) - codedict['pose'][:,:3] = global_pose - for (i,k) in enumerate(range(0,31,2)): #jaw angle from -50 to 50 + global_pose = batch_euler2axis(deg2rad(euler_pose[:, :3].cuda())) + codedict['pose'][:, :3] = global_pose + for (i, k) in enumerate(range(0, 31, 2)): # jaw angle from -50 to 50 # expression: jaw pose euler_pose = torch.randn((1, 3)) - euler_pose[:,0] = k#torch.rand((self.batch_size))*160 - 80 - euler_pose[:,1] = 0#(torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) - euler_pose[:,2] = 0#(torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) - jaw_pose = batch_euler2axis(deg2rad(euler_pose[:,:3].cuda())) - codedict['pose'][:,3:] = jaw_pose - _, visdict_view = deca.decode(codedict) + euler_pose[:, 0] = k # torch.rand((self.batch_size))*160 - 80 + euler_pose[:, 1] = 0 # (torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) + euler_pose[:, 2] = 0 # (torch.rand((self.batch_size))*60 - 30)*(2./euler_pose[:,1].abs()) + jaw_pose = batch_euler2axis(deg2rad(euler_pose[:, :3].cuda())) + codedict['pose'][:, 3:] = jaw_pose + _, visdict_view = deca.decode(codedict) visdict_list[i]['exp'] = visdict_view['shape_detail_images'] count = i - for (i,k) in enumerate(range(len(expdata))): #jaw angle from -50 to 50 + for (i, k) in enumerate(range(len(expdata))): # jaw angle from -50 to 50 # expression: jaw pose - exp_images = expdata[i]['image'].to(device)[None,...] + exp_images = expdata[i]['image'].to(device)[None, ...] exp_codedict = deca.encode(exp_images) # transfer exp code - codedict['pose'][:,3:] = exp_codedict['pose'][:,3:] + codedict['pose'][:, 3:] = exp_codedict['pose'][:, 3:] codedict['exp'] = exp_codedict['exp'] _, exp_visdict = deca.decode(codedict) - visdict_list[i+count]['exp'] = exp_visdict['shape_detail_images'] + visdict_list[i + count]['exp'] = exp_visdict['shape_detail_images'] visdict_list_list.append(visdict_list) - + ### write gif writer = imageio.get_writer(os.path.join(savefolder, 'teaser.gif'), mode='I') for i in range(len(yaw_list)): @@ -103,30 +104,31 @@ def main(args): grid_image = deca.visualize(visdict_list_list[j][i]) grid_image_list.append(grid_image) grid_image_all = np.concatenate(grid_image_list, 0) - grid_image_all = rescale(grid_image_all, 0.6, multichannel=True) # resize for showing in github - writer.append_data(grid_image_all[:,:,[2,1,0]]) + grid_image_all = rescale(grid_image_all, 0.6, channel_axis=-1) # resize for showing in github + grid_image_all = np.clip(grid_image_all * 255, 0, 255).astype(np.uint8) + writer.append_data(grid_image_all[:, :, [2, 1, 0]]) print(f'-- please check the teaser figure in {savefolder}') - + if __name__ == '__main__': parser = argparse.ArgumentParser(description='DECA: Detailed Expression Capture and Animation') parser.add_argument('-i', '--inputpath', default='TestSamples/teaser', type=str, help='path to the test data, can be image folder, image path, image list, video') - parser.add_argument('-e', '--exp_path', default='TestSamples/exp', type=str, + parser.add_argument('-e', '--exp_path', default='TestSamples/exp', type=str, help='path to expression') parser.add_argument('-s', '--savefolder', default='TestSamples/teaser/results', type=str, help='path to the output directory, where results(obj, txt files) will be stored.') parser.add_argument('--device', default='cuda', type=str, - help='set device, cpu for using cpu' ) + help='set device, cpu for using cpu') # rendering option parser.add_argument('--rasterizer_type', default='standard', type=str, - help='rasterizer type: pytorch3d or standard' ) + help='rasterizer type: pytorch3d or standard') # process test images parser.add_argument('--iscrop', default=True, type=lambda x: x.lower() in ['true', '1'], - help='whether to crop input image, set false only when the test image are well cropped' ) + help='whether to crop input image, set false only when the test image are well cropped') parser.add_argument('--detector', default='fan', type=str, - help='detector for cropping face, check detectos.py for details' ) + help='detector for cropping face, check detectos.py for details') - main(parser.parse_args()) \ No newline at end of file + main(parser.parse_args()) diff --git a/demos/demo_transfer.py b/demos/demo_transfer.py index 13d8441a..246930f1 100755 --- a/demos/demo_transfer.py +++ b/demos/demo_transfer.py @@ -32,8 +32,8 @@ def main(args): os.makedirs(savefolder, exist_ok=True) # load test images - testdata = datasets.TestData(args.image_path, iscrop=args.iscrop, face_detector=args.detector) - expdata = datasets.TestData(args.exp_path, iscrop=args.iscrop, face_detector=args.detector) + testdata = datasets.TestData(args.image_path, iscrop=args.iscrop, face_detector=args.detector, device=args.device) + expdata = datasets.TestData(args.exp_path, iscrop=args.iscrop, face_detector=args.detector, device=args.device) # run DECA deca_cfg.model.use_tex = args.useTex diff --git a/fetch_data.sh b/fetch_data.sh index 65c3bd22..c893ab45 100644 --- a/fetch_data.sh +++ b/fetch_data.sh @@ -14,10 +14,5 @@ echo -e "\nDownloading FLAME..." wget --post-data "username=$username&password=$password" 'https://download.is.tue.mpg.de/download.php?domain=flame&sfile=FLAME2020.zip&resume=1' -O './data/FLAME2020.zip' --no-check-certificate --continue unzip ./data/FLAME2020.zip -d ./data/FLAME2020 mv ./data/FLAME2020/generic_model.pkl ./data -# rm -rf ./models -echo -e "\nDownloading deca_model..." - -FILEID=1rp8kdyLPvErw2dTmqtjISRVvQLj6Yzje -FILENAME=./data/deca_model.tar -wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id='${FILEID} -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=${FILEID}" -O $FILENAME && rm -rf /tmp/cookies.txt \ No newline at end of file +echo -e "\nDownload deca_model from https://drive.usercontent.google.com/download?id=1rp8kdyLPvErw2dTmqtjISRVvQLj6Yzje&export=download&authuser=0 and move it to data folder" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 455f1732..62d6d9e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,16 @@ -#to install all this requirements +# to install all this requirements #`pip install -r requirements.txt` -numpy>=1.18.5 -scipy>=1.4.1 +numpy==1.23.5 +scipy==1.9.3 chumpy>=0.69 -scikit-image>=0.15 -opencv-python>=4.1.1 -scikit-image>=0.15 #skimage -PyYAML==5.1.1 -torch==1.6.0 # for compatible with pytorch3d -torchvision==0.7.0 +scikit-image>=0.15 +opencv-python +scikit-image>=0.15 +#skimage +PyYAML face-alignment yacs==0.1.8 -kornia==0.4.0 +kornia ninja fvcore -# pytorch3d \ No newline at end of file +# pytorch3d \ No newline at end of file