This repository has been archived by the owner on Jul 7, 2022. It is now read-only.
forked from vsitzmann/deepvoxels
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdeep_voxels.py
173 lines (147 loc) · 8.15 KB
/
deep_voxels.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
import torch
import torch.nn as nn
import numpy as np
from projection import *
from custom_layers import *
import functools
import util
from pytorch_prototyping.pytorch_prototyping import *
class DeepVoxels(nn.Module):
def __init__(self,
img_sidelength,
lifting_img_dims,
frustrum_img_dims,
grid_dims,
num_grid_feats=64,
nf0=64,
use_occlusion_net=True):
''' Initializes the DeepVoxels model.
:param img_sidelength: The sidelength of the input images (for instance 512)
:param lifting_img_dims: The dimensions of the feature map to be lifted.
:param frustrum_img_dims: The dimensions of the canonical view volume that DeepVoxels are resampled to.
:param grid_dims: The dimensions of the deepvoxels grid.
:param grid_dims: The number of featres in the outermost layer of U-Nets.
:param use_occlusion_net: Whether to use the OcclusionNet or not.
'''
super().__init__()
self.use_occlusion_net = use_occlusion_net
self.grid_dims = grid_dims
self.norm = nn.BatchNorm2d
self.lifting_img_dims = lifting_img_dims
self.frustrum_img_dims = frustrum_img_dims
self.grid_dims = grid_dims
# The frustrum depth is the number of voxels in the depth dimension of the canonical viewing volume.
# It's calculated as the length of the diagonal of the DeepVoxels grid.
self.frustrum_depth = 2 * grid_dims[-1]
self.nf0 = nf0 # Number of features to use in the outermost layer of all U-Nets
self.n_grid_feats = num_grid_feats # Number of features in the DeepVoxels grid.
self.occnet_nf = 4 # Number of features to use in the 3D unet of the occlusion subnetwork
# Feature extractor is an asymmetric UNet: Straight downsampling to 64x64, then a UNet with skip connections
self.feature_extractor = FeatureExtractor(nf0=self.nf0,
out_channels=self.n_grid_feats,
input_resolution=img_sidelength,
output_sidelength=self.frustrum_img_dims[0])
# Rendering net is an asymmetric UNet: UNet with skip connections and then straight upsampling
self.rendering_net = RenderingNet(nf0=self.nf0,
in_channels=self.n_grid_feats,
input_resolution=self.frustrum_img_dims[0],
img_sidelength=img_sidelength)
if self.use_occlusion_net:
self.occlusion_net = OcclusionNet(nf0=self.n_grid_feats,
occnet_nf=self.occnet_nf,
frustrum_dims=[self.frustrum_img_dims[0], self.frustrum_img_dims[1],
self.frustrum_depth])
print(self.occlusion_net)
else:
self.depth_collapse_net = nn.Sequential(
Conv2dSame(self.n_grid_feats * self.frustrum_depth,
out_channels=self.nf0 * self.grid_dims[-1] // 2,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 2),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 2,
out_channels=self.nf0 * self.grid_dims[-1] // 8,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 8),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 8,
out_channels=self.nf0,
kernel_size=3,
bias=False),
self.norm(self.nf0),
nn.ReLU(True),
)
print(self.depth_collapse_net)
# The deepvoxels grid is registered as a buffer - meaning, it is safed together with model parameters, but is
# not trainable.
self.register_buffer("deepvoxels",
torch.zeros(
(1, self.n_grid_feats, self.grid_dims[0], self.grid_dims[1], self.grid_dims[2])))
self.integration_net = IntegrationNet(self.n_grid_feats,
use_dropout=True,
coord_conv=True,
per_feature=False,
grid_dim=grid_dims[-1])
self.inpainting_net = Unet3d(in_channels=self.n_grid_feats + 3,
out_channels=self.n_grid_feats,
num_down=2,
nf0=self.n_grid_feats,
max_channels=4 * self.n_grid_feats)
print(100 * "*")
print("inpainting_net")
util.print_network(self.inpainting_net)
print(self.inpainting_net)
print("rendering net")
util.print_network(self.rendering_net)
print(self.rendering_net)
print("feature extraction net")
util.print_network(self.feature_extractor)
print(self.feature_extractor)
print(100 * "*")
# Coordconv volumes
coord_conv_volume = np.mgrid[-self.grid_dims[0] // 2:self.grid_dims[0] // 2,
-self.grid_dims[1] // 2:self.grid_dims[1] // 2,
-self.grid_dims[2] // 2:self.grid_dims[2] // 2]
coord_conv_volume = np.stack(coord_conv_volume, axis=0).astype(np.float32)
coord_conv_volume = coord_conv_volume / self.grid_dims[0]
self.coord_conv_volume = torch.Tensor(coord_conv_volume).float().cuda()[None, :, :, :, :]
def forward(self,
input_img,
proj_frustrum_idcs_list,
proj_grid_coords_list,
lift_volume_idcs,
lift_img_coords,
writer):
if input_img is not None:
# Training mode: Extract features from input img, lift them, and update the deepvoxels volume.
img_feats = self.feature_extractor(input_img)
temp_feat_vol = interpolate_lifting(img_feats, lift_volume_idcs, lift_img_coords, self.grid_dims)
dv_new = self.integration_net(temp_feat_vol, self.deepvoxels.detach(), writer)
self.deepvoxels.data = dv_new
else:
# Testing mode: Use the pre-trained deepvoxels volume.
dv_new = self.deepvoxels
inpainting_input = torch.cat([dv_new, self.coord_conv_volume], dim=1)
dv_inpainted = self.inpainting_net(inpainting_input)
novel_views, depth_maps = list(), list()
for i, (proj_frustrum_idcs, proj_grid_coords) in enumerate(zip(proj_frustrum_idcs_list, proj_grid_coords_list)):
can_view_vol = interpolate_trilinear(dv_inpainted,
proj_frustrum_idcs,
proj_grid_coords,
self.frustrum_img_dims,
self.frustrum_depth)
if self.use_occlusion_net:
visibility_weights, depth_map = self.occlusion_net(can_view_vol)
depth_maps.append(depth_map)
collapsed_frustrum = torch.mean(visibility_weights * can_view_vol, dim=2)
novel_image_features = collapsed_frustrum.contiguous().view(
[1, -1, self.frustrum_img_dims[0], self.frustrum_img_dims[1]])
else:
frustrum_collapse_input = can_view_vol.view(1, -1, self.frustrum_img_dims[0], self.frustrum_img_dims[1])
novel_image_features = self.depth_collapse_net(frustrum_collapse_input)
depth_maps.append(torch.zeros((1, 1, 64, 64)))
rendered_img = 0.5 * self.rendering_net(novel_image_features)
novel_views.append(rendered_img)
return novel_views, depth_maps