trellis2/renderers/mesh_renderer.py

from typing import *
import torch
from easydict import EasyDict as edict
from ..representations.mesh import Mesh, MeshWithVoxel, MeshWithPbrMaterial, TextureFilterMode, AlphaMode, TextureWrapMode
import torch.nn.functional as F


def intrinsics_to_projection(
        intrinsics: torch.Tensor,
        near: float,
        far: float,
    ) -> torch.Tensor:
    """
    OpenCV intrinsics to OpenGL perspective matrix

    Args:
        intrinsics (torch.Tensor): [3, 3] OpenCV intrinsics matrix
        near (float): near plane to clip
        far (float): far plane to clip
    Returns:
        (torch.Tensor): [4, 4] OpenGL perspective matrix
    """
    fx, fy = intrinsics[0, 0], intrinsics[1, 1]
    cx, cy = intrinsics[0, 2], intrinsics[1, 2]
    ret = torch.zeros((4, 4), dtype=intrinsics.dtype, device=intrinsics.device)
    ret[0, 0] = 2 * fx
    ret[1, 1] = 2 * fy
    ret[0, 2] = 2 * cx - 1
    ret[1, 2] = - 2 * cy + 1
    ret[2, 2] = (far + near) / (far - near)
    ret[2, 3] = 2 * near * far / (near - far)
    ret[3, 2] = 1.
    return ret
    

class MeshRenderer:
    """
    Renderer for the Mesh representation.

    Args:
        rendering_options (dict): Rendering options.
        """
    def __init__(self, rendering_options={}, device='cuda'):
        if 'dr' not in globals():
            import nvdiffrast.torch as dr
        
        self.rendering_options = edict({
            "resolution": None,
            "near": None,
            "far": None,
            "ssaa": 1,
            "chunk_size": None,
            "antialias": True,
            "clamp_barycentric_coords": False,
        })
        self.rendering_options.update(rendering_options)
        self.glctx = dr.RasterizeCudaContext(device=device)
        self.device=device
        
    def render(
            self,
            mesh : Mesh,
            extrinsics: torch.Tensor,
            intrinsics: torch.Tensor,
            return_types = ["mask", "normal", "depth"],
            transformation : Optional[torch.Tensor] = None
        ) -> edict:
        """
        Render the mesh.

        Args:
            mesh : meshmodel
            extrinsics (torch.Tensor): (4, 4) camera extrinsics
            intrinsics (torch.Tensor): (3, 3) camera intrinsics
            return_types (list): list of return types, can be "attr", "mask", "depth", "coord", "normal"

        Returns:
            edict based on return_types containing:
                attr (torch.Tensor): [C, H, W] rendered attr image
                depth (torch.Tensor): [H, W] rendered depth image
                normal (torch.Tensor): [3, H, W] rendered normal image
                mask (torch.Tensor): [H, W] rendered mask image
        """
        if 'dr' not in globals():
            import nvdiffrast.torch as dr
            
        resolution = self.rendering_options["resolution"]
        near = self.rendering_options["near"]
        far = self.rendering_options["far"]
        ssaa = self.rendering_options["ssaa"]
        chunk_size = self.rendering_options["chunk_size"]
        antialias = self.rendering_options["antialias"]
        clamp_barycentric_coords = self.rendering_options["clamp_barycentric_coords"]
        
        if mesh.vertices.shape[0] == 0 or mesh.faces.shape[0] == 0:
            ret_dict = edict()
            for type in return_types:
                if type == "mask" :
                    ret_dict[type] = torch.zeros((resolution, resolution), dtype=torch.float32, device=self.device)
                elif type == "depth":
                    ret_dict[type] = torch.zeros((resolution, resolution), dtype=torch.float32, device=self.device)
                elif type == "normal":
                    ret_dict[type] = torch.full((3, resolution, resolution), 0.5, dtype=torch.float32, device=self.device)
                elif type == "coord":
                    ret_dict[type] = torch.zeros((3, resolution, resolution), dtype=torch.float32, device=self.device)
                elif type == "attr":
                    if isinstance(mesh, MeshWithVoxel):
                        ret_dict[type] = torch.zeros((mesh.attrs.shape[-1], resolution, resolution), dtype=torch.float32, device=self.device)
                    else:
                        ret_dict[type] = torch.zeros((mesh.vertex_attrs.shape[-1], resolution, resolution), dtype=torch.float32, device=self.device)
            return ret_dict
        
        perspective = intrinsics_to_projection(intrinsics, near, far)
        
        full_proj = (perspective @ extrinsics).unsqueeze(0)
        extrinsics = extrinsics.unsqueeze(0)
        
        vertices = mesh.vertices.unsqueeze(0)
        vertices_homo = torch.cat([vertices, torch.ones_like(vertices[..., :1])], dim=-1)
        if transformation is not None:
            vertices_homo = torch.bmm(vertices_homo, transformation.unsqueeze(0).transpose(-1, -2))
            vertices = vertices_homo[..., :3].contiguous()
        vertices_camera = torch.bmm(vertices_homo, extrinsics.transpose(-1, -2))
        vertices_clip = torch.bmm(vertices_homo, full_proj.transpose(-1, -2))
        faces = mesh.faces
        
        if 'normal' in return_types:
            v0 = vertices_camera[0, mesh.faces[:, 0], :3]
            v1 = vertices_camera[0, mesh.faces[:, 1], :3]
            v2 = vertices_camera[0, mesh.faces[:, 2], :3]
            e0 = v1 - v0
            e1 = v2 - v0
            face_normal = torch.cross(e0, e1, dim=1)
            face_normal = F.normalize(face_normal, dim=1)
            face_normal = torch.where(torch.sum(face_normal * v0, dim=1, keepdim=True) > 0, face_normal, -face_normal)
        
        out_dict = edict()
        if chunk_size is None:
            rast, rast_db = dr.rasterize(
                self.glctx, vertices_clip, faces, (resolution * ssaa, resolution * ssaa)
            )
            if clamp_barycentric_coords:
                rast[..., :2] = torch.clamp(rast[..., :2], 0, 1)
                rast[..., :2] /= torch.where(rast[..., :2].sum(dim=-1, keepdim=True) > 1, rast[..., :2].sum(dim=-1, keepdim=True), torch.ones_like(rast[..., :2]))
            for type in return_types:
                img = None
                if type == "mask" :
                    img = (rast[..., -1:] > 0).float()
                    if antialias: img = dr.antialias(img, rast, vertices_clip, faces)
                elif type == "depth":
                    img = dr.interpolate(vertices_camera[..., 2:3].contiguous(), rast, faces)[0]
                    if antialias: img = dr.antialias(img, rast, vertices_clip, faces)
                elif type == "normal" :
                    img = dr.interpolate(face_normal.unsqueeze(0), rast, torch.arange(face_normal.shape[0], dtype=torch.int, device=self.device).unsqueeze(1).repeat(1, 3).contiguous())[0]
                    if antialias: img = dr.antialias(img, rast, vertices_clip, faces)
                    img = (img + 1) / 2
                elif type == "coord":
                    img = dr.interpolate(vertices, rast, faces)[0]
                    if antialias: img = dr.antialias(img, rast, vertices_clip, faces)
                elif type == "attr":
                    if isinstance(mesh, MeshWithVoxel):
                        if 'grid_sample_3d' not in globals():
                            from flex_gemm.ops.grid_sample import grid_sample_3d
                        mask = rast[..., -1:] > 0
                        xyz = dr.interpolate(vertices, rast, faces)[0]
                        xyz = ((xyz - mesh.origin) / mesh.voxel_size).reshape(1, -1, 3)
                        img = grid_sample_3d(
                            mesh.attrs,
                            torch.cat([torch.zeros_like(mesh.coords[..., :1]), mesh.coords], dim=-1),
                            mesh.voxel_shape,
                            xyz,
                            mode='trilinear'
                        )
                        img = img.reshape(1, resolution * ssaa, resolution * ssaa, mesh.attrs.shape[-1]) * mask
                    elif isinstance(mesh, MeshWithPbrMaterial):
                        tri_id = rast[0, :, :, -1:]
                        mask = tri_id > 0
                        uv_coords = mesh.uv_coords.reshape(1, -1, 2)
                        texc, texd = dr.interpolate(
                            uv_coords,
                            rast,
                            torch.arange(mesh.uv_coords.shape[0] * 3, dtype=torch.int, device=self.device).reshape(-1, 3),
                            rast_db=rast_db,
                            diff_attrs='all'
                        )
                        # Fix problematic texture coordinates
                        texc = torch.nan_to_num(texc, nan=0.0, posinf=1e3, neginf=-1e3)
                        texc = torch.clamp(texc, min=-1e3, max=1e3)
                        texd = torch.nan_to_num(texd, nan=0.0, posinf=1e3, neginf=-1e3)
                        texd = torch.clamp(texd, min=-1e3, max=1e3)
                        mid = mesh.material_ids[(tri_id - 1).long()]
                        imgs = {
                            'base_color': torch.zeros((resolution * ssaa, resolution * ssaa, 3), dtype=torch.float32, device=self.device),
                            'metallic': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device),
                            'roughness': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device),
                            'alpha': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device)
                        }
                        for id, mat in enumerate(mesh.materials):
                            mat_mask = (mid == id).float() * mask.float()
                            mat_texc = texc * mat_mask
                            mat_texd = texd * mat_mask

                            if mat.base_color_texture is not None:
                                base_color = dr.texture(
                                    mat.base_color_texture.image.unsqueeze(0),
                                    mat_texc,
                                    mat_texd,
                                    filter_mode='linear-mipmap-linear' if mat.base_color_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                    boundary_mode='clamp' if mat.base_color_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                )[0]
                                imgs['base_color'] += base_color * mat.base_color_factor * mat_mask
                            else:
                                imgs['base_color'] += mat.base_color_factor * mat_mask
                                
                            if mat.metallic_texture is not None:
                                metallic = dr.texture(
                                    mat.metallic_texture.image.unsqueeze(0),
                                    mat_texc,
                                    mat_texd,
                                    filter_mode='linear-mipmap-linear' if mat.metallic_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                    boundary_mode='clamp' if mat.metallic_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                )[0]
                                imgs['metallic'] += metallic * mat.metallic_factor * mat_mask
                            else:
                                imgs['metallic'] += mat.metallic_factor * mat_mask

                            if mat.roughness_texture is not None:
                                roughness = dr.texture(
                                    mat.roughness_texture.image.unsqueeze(0),
                                    mat_texc,
                                    mat_texd,
                                    filter_mode='linear-mipmap-linear' if mat.roughness_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                    boundary_mode='clamp' if mat.roughness_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                )[0]
                                imgs['roughness'] += roughness * mat.roughness_factor * mat_mask
                            else:
                                imgs['roughness'] += mat.roughness_factor * mat_mask

                            if mat.alpha_mode == AlphaMode.OPAQUE:
                                imgs['alpha'] += 1.0 * mat_mask
                            else:
                                if mat.alpha_texture is not None:
                                    alpha = dr.texture(
                                        mat.alpha_texture.image.unsqueeze(0),
                                        mat_texc,
                                        mat_texd,
                                        filter_mode='linear-mipmap-linear' if mat.alpha_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                        boundary_mode='clamp' if mat.alpha_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                    )[0]
                                    if mat.alpha_mode == AlphaMode.MASK:
                                        imgs['alpha'] += (alpha * mat.alpha_factor > mat.alpha_cutoff).float() * mat_mask
                                    elif mat.alpha_mode == AlphaMode.BLEND:
                                        imgs['alpha'] += alpha * mat.alpha_factor * mat_mask
                                else:
                                    if mat.alpha_mode == AlphaMode.MASK:
                                        imgs['alpha'] += (mat.alpha_factor > mat.alpha_cutoff).float() * mat_mask
                                    elif mat.alpha_mode == AlphaMode.BLEND:
                                        imgs['alpha'] += mat.alpha_factor * mat_mask
                    
                        img = torch.cat([imgs[name] for name in imgs.keys()], dim=-1).unsqueeze(0)
                    else:
                        img = dr.interpolate(mesh.vertex_attrs.unsqueeze(0), rast, faces)[0]
                        if antialias: img = dr.antialias(img, rast, vertices_clip, faces)
                        
                out_dict[type] = img
        else:
            z_buffer = torch.full((1, resolution * ssaa, resolution * ssaa), torch.inf, device=self.device, dtype=torch.float32)
            for i in range(0, faces.shape[0], chunk_size):
                faces_chunk = faces[i:i+chunk_size]
                rast, rast_db = dr.rasterize(
                    self.glctx, vertices_clip, faces_chunk, (resolution * ssaa, resolution * ssaa)
                )
                z_filter = torch.logical_and(
                    rast[..., 3] != 0,
                    rast[..., 2] < z_buffer
                )
                z_buffer[z_filter] = rast[z_filter][..., 2]
            
                for type in return_types:
                    img = None
                    if type == "mask" :
                        img = (rast[..., -1:] > 0).float()
                    elif type == "depth":
                        img = dr.interpolate(vertices_camera[..., 2:3].contiguous(), rast, faces_chunk)[0]
                    elif type == "normal" :
                        face_normal_chunk = face_normal[i:i+chunk_size]
                        img = dr.interpolate(face_normal_chunk.unsqueeze(0), rast, torch.arange(face_normal_chunk.shape[0], dtype=torch.int, device=self.device).unsqueeze(1).repeat(1, 3).contiguous())[0]
                        img = (img + 1) / 2
                    elif type == "coord":
                        img = dr.interpolate(vertices, rast, faces_chunk)[0]
                    elif type == "attr":
                        if isinstance(mesh, MeshWithVoxel):
                            if 'grid_sample_3d' not in globals():
                                from flex_gemm.ops.grid_sample import grid_sample_3d
                            mask = rast[..., -1:] > 0
                            xyz = dr.interpolate(vertices, rast, faces_chunk)[0]
                            xyz = ((xyz - mesh.origin) / mesh.voxel_size).reshape(1, -1, 3)
                            img = grid_sample_3d(
                                mesh.attrs,
                                torch.cat([torch.zeros_like(mesh.coords[..., :1]), mesh.coords], dim=-1),
                                mesh.voxel_shape,
                                xyz,
                                mode='trilinear'
                            )
                            img = img.reshape(1, resolution * ssaa, resolution * ssaa, mesh.attrs.shape[-1]) * mask
                        elif isinstance(mesh, MeshWithPbrMaterial):
                            tri_id = rast[0, :, :, -1:]
                            mask = tri_id > 0
                            uv_coords = mesh.uv_coords.reshape(1, -1, 2)
                            texc, texd = dr.interpolate(
                                uv_coords,
                                rast,
                                torch.arange(mesh.uv_coords.shape[0] * 3, dtype=torch.int, device=self.device).reshape(-1, 3),
                                rast_db=rast_db,
                                diff_attrs='all'
                            )
                            # Fix problematic texture coordinates
                            texc = torch.nan_to_num(texc, nan=0.0, posinf=1e3, neginf=-1e3)
                            texc = torch.clamp(texc, min=-1e3, max=1e3)
                            texd = torch.nan_to_num(texd, nan=0.0, posinf=1e3, neginf=-1e3)
                            texd = torch.clamp(texd, min=-1e3, max=1e3)
                            mid = mesh.material_ids[(tri_id - 1).long()]
                            imgs = {
                                'base_color': torch.zeros((resolution * ssaa, resolution * ssaa, 3), dtype=torch.float32, device=self.device),
                                'metallic': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device),
                                'roughness': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device),
                                'alpha': torch.zeros((resolution * ssaa, resolution * ssaa, 1), dtype=torch.float32, device=self.device)
                            }
                            for id, mat in enumerate(mesh.materials):
                                mat_mask = (mid == id).float() * mask.float()
                                mat_texc = texc * mat_mask
                                mat_texd = texd * mat_mask

                                if mat.base_color_texture is not None:
                                    base_color = dr.texture(
                                        mat.base_color_texture.image.unsqueeze(0),
                                        mat_texc,
                                        mat_texd,
                                        filter_mode='linear-mipmap-linear' if mat.base_color_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                        boundary_mode='clamp' if mat.base_color_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                    )[0]
                                    imgs['base_color'] += base_color * mat.base_color_factor * mat_mask
                                else:
                                    imgs['base_color'] += mat.base_color_factor * mat_mask
                                    
                                if mat.metallic_texture is not None:
                                    metallic = dr.texture(
                                        mat.metallic_texture.image.unsqueeze(0),
                                        mat_texc,
                                        mat_texd,
                                        filter_mode='linear-mipmap-linear' if mat.metallic_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                        boundary_mode='clamp' if mat.metallic_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                    )[0]
                                    imgs['metallic'] += metallic * mat.metallic_factor * mat_mask
                                else:
                                    imgs['metallic'] += mat.metallic_factor * mat_mask

                                if mat.roughness_texture is not None:
                                    roughness = dr.texture(
                                        mat.roughness_texture.image.unsqueeze(0),
                                        mat_texc,
                                        mat_texd,
                                        filter_mode='linear-mipmap-linear' if mat.roughness_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                        boundary_mode='clamp' if mat.roughness_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                    )[0]
                                    imgs['roughness'] += roughness * mat.roughness_factor * mat_mask
                                else:
                                    imgs['roughness'] += mat.roughness_factor * mat_mask

                                if mat.alpha_mode == AlphaMode.OPAQUE:
                                    imgs['alpha'] += 1.0 * mat_mask
                                else:
                                    if mat.alpha_texture is not None:
                                        alpha = dr.texture(
                                            mat.alpha_texture.image.unsqueeze(0),
                                            mat_texc,
                                            mat_texd,
                                            filter_mode='linear-mipmap-linear' if mat.alpha_texture.filter_mode == TextureFilterMode.LINEAR else 'nearest',
                                            boundary_mode='clamp' if mat.alpha_texture.wrap_mode == TextureWrapMode.CLAMP_TO_EDGE else 'wrap'
                                        )[0]
                                        if mat.alpha_mode == AlphaMode.MASK:
                                            imgs['alpha'] += (alpha * mat.alpha_factor > mat.alpha_cutoff).float() * mat_mask
                                        elif mat.alpha_mode == AlphaMode.BLEND:
                                            imgs['alpha'] += alpha * mat.alpha_factor * mat_mask
                                    else:
                                        if mat.alpha_mode == AlphaMode.MASK:
                                            imgs['alpha'] += (mat.alpha_factor > mat.alpha_cutoff).float() * mat_mask
                                        elif mat.alpha_mode == AlphaMode.BLEND:
                                            imgs['alpha'] += mat.alpha_factor * mat_mask
                        
                            img = torch.cat([imgs[name] for name in imgs.keys()], dim=-1).unsqueeze(0)
                        else:
                            img = dr.interpolate(mesh.vertex_attrs.unsqueeze(0), rast, faces_chunk)[0]
                            
                    if type not in out_dict:
                        out_dict[type] = img
                    else:
                        out_dict[type][z_filter] = img[z_filter]

        for type in return_types:
            img = out_dict[type]
            if ssaa > 1:
                img = F.interpolate(img.permute(0, 3, 1, 2), (resolution, resolution), mode='bilinear', align_corners=False, antialias=True)
                img = img.squeeze()
            else:
                img = img.permute(0, 3, 1, 2).squeeze()
            out_dict[type] = img

        if isinstance(mesh, (MeshWithVoxel, MeshWithPbrMaterial)) and 'attr' in return_types:
            for k, s in mesh.layout.items():
                out_dict[k] = out_dict['attr'][s]
            del out_dict['attr']
        
        return out_dict