3d Earth Models From Multi-view Satellite Images

📆 Project Period	October, 2023
📍 GitHub	gitlab.esa.int Project Summary

Shadow Neural Radiance Fields demonstration notebook

This notebook contains the demonstration of how to render plots from a pre-trained S-NeRF model, produced using snerf/train.py. It is based on the use of TensorFlow-2.2.0 with a single CUDA-enabled GPU.

import os
import math
import numpy as np

import tensorflow as tf
os.environ['CUDA_VISIBLE_DEVICES'] = '1'

import matplotlib.pyplot as plt
import pprint

from IPython.display import HTML, display

# from snerf import data_handling
# from snerf import models
# from snerf import render
# from snerf import train
# from snerf import plots

import data_handling
import models
import render
import train
import plots

import os

np.set_printoptions(precision=3, suppress=True)
def_dtype = np.float32

print(tf.__version__)
#physical_devices = tf.config.experimental.list_physical_devices('GPU')
physical_devices = tf.config.experimental.list_physical_devices('GPU 0')
print(physical_devices)
print(os.getcwd())

2.6.0
[]
/home/jovyan/work/SNERF/notebooks

# Read config
parser = train.config_parser()
#args = parser.parse_args('--config configs/068/siren_df2_full_sc_config.txt')
args = parser.parse_args('--config ../configs/068_config.txt')
arg_dict = vars(args)
pprint.pprint(arg_dict)
dataset = data_handling.generate_dataset(arg_dict)

{'config': '../configs/068_config.txt',
 'data.depth.df': 1,
 'data.depth.path': '/home/jovyan/eodata/data/068/JAX_068_df1_dsm.tif',
 'data.image.df': 8,
 'data.image.path': '/home/jovyan/eodata/data/068/JAX_068_df1',
 'data.image.sd': 0.3,
 'data.md.path': '/home/jovyan/eodata/data/068/JAX_068_df1_md.txt',
 'data.test_id': ['20', '22'],
 'data.train_id': ['01',
                   '02',
                   '03',
                   '04',
                   '05',
                   '06',
                   '07',
                   '09',
                   '10',
                   '11',
                   '12',
                   '13',
                   '14',
                   '15',
                   '18',
                   '19'],
 'gpu': '0',
 'model.act': 'sin',
 'model.act.sin.w0': 32.0,
 'model.c.depth': 1,
 'model.c.width': 50,
 'model.emb.dir': 1,
 'model.emb.pos': 0,
 'model.ins.light': True,
 'model.ins.views': False,
 'model.outs.shad': True,
 'model.outs.sky': True,
 'model.shad.depth': 4,
 'model.shad.width': 50,
 'model.sigma.depth': 8,
 'model.sigma.skips': [],
 'model.sigma.width': 100,
 'out.iplot': 1000,
 'out.path': '../results/068/',
 'rend.mode': 'alt',
 'rend.mode.alt.max': 30.0,
 'rend.mode.alt.min': -30.0,
 'rend.mode.nf.far': 10.0,
 'rend.mode.nf.near': 3.0,
 'rend.nimportance': 64,
 'rend.nsamples': 64,
 'rend.rescale': None,
 'rend.unzoom': True,
 'train.lr.decay': 0.1,
 'train.lr.init': 1e-04,
 'train.n_epoch': 10000,
 'train.n_rand': 256,
 'train.noise.shad': 1.0,
 'train.noise.sigma': 10.0,
 'train.shad': True,
 'train.shad.custom': 'linear',
 'train.shad.custom.bounds.end': [100.0, 80.0],
 'train.shad.custom.bounds.samp': [30, 1],
 'train.shad.custom.bounds.start': [160.0, 40.0],
 'train.shad.df': 1,
 'train.shad.lambda': 0.05}
/home/jovyan/eodata/data/068/JAX_068_df1_01.tif
/home/jovyan/eodata/data/068/JAX_068_df1_02.tif
/home/jovyan/eodata/data/068/JAX_068_df1_03.tif
/home/jovyan/eodata/data/068/JAX_068_df1_04.tif
/home/jovyan/eodata/data/068/JAX_068_df1_05.tif
/home/jovyan/eodata/data/068/JAX_068_df1_06.tif
/home/jovyan/eodata/data/068/JAX_068_df1_07.tif
/home/jovyan/eodata/data/068/JAX_068_df1_09.tif
/home/jovyan/eodata/data/068/JAX_068_df1_10.tif
/home/jovyan/eodata/data/068/JAX_068_df1_11.tif
/home/jovyan/eodata/data/068/JAX_068_df1_12.tif
/home/jovyan/eodata/data/068/JAX_068_df1_13.tif
/home/jovyan/eodata/data/068/JAX_068_df1_14.tif
/home/jovyan/eodata/data/068/JAX_068_df1_15.tif
/home/jovyan/eodata/data/068/JAX_068_df1_18.tif
/home/jovyan/eodata/data/068/JAX_068_df1_19.tif
/home/jovyan/eodata/data/068/JAX_068_df1_20.tif
/home/jovyan/eodata/data/068/JAX_068_df1_22.tif

Here we visualize the image set used for training the network, as well as the corresponding light and viewing angles for the scene. We also show the Ground Truth DEM of the scene (airborne-lidar based), which is used for evaluation of the altitude estimation.

plots.plot_images(arg_dict['data.train_id'], dataset['train_imgs'], dataset['train_view_dirs'], dataset['train_light_dirs'])

plots.plot_view_light_directions(dataset['train_view_dirs'], dataset['train_light_dirs'])

plots.plot_depth_map(dataset['depth_map'])

We load the pre-trained model and show its internal architecture, including the number of trainable parameters.

model = models.load_model(f"../models/068/model.npy", arg_dict)
model['model'].summary()

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to
==================================================================================================
input_1 (InputLayer)            [(None, 7)]          0
__________________________________________________________________________________________________
tf.split (TFOpLambda)           [(None, 3), (None, 4 0           input_1[0][0]
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          400         tf.split[0][0]
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense[0
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_1
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_2
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_3
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_4
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_5
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 100)          10100       sinusodial_representation_dense_6
__________________________________________________________________________________________________
dense (Dense)                   (None, 100)          10100       sinusodial_representation_dense_7
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 50)           250         tf.split[0][1]
__________________________________________________________________________________________________
tf.concat_1 (TFOpLambda)        (None, 150)          0           dense[0][0]
                                                                 sinusodial_representation_dense_9
__________________________________________________________________________________________________
dense_3 (Dense)                 (None, 50)           7550        tf.concat_1[0][0]
__________________________________________________________________________________________________
dense_4 (Dense)                 (None, 50)           2550        dense_3[0][0]
__________________________________________________________________________________________________
sinusodial_representation_dense (None, 50)           5050        dense[0][0]
__________________________________________________________________________________________________
dense_5 (Dense)                 (None, 50)           2550        dense_4[0][0]
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 3)            153         sinusodial_representation_dense_8
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 1)            101         dense[0][0]
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 50)           2550        dense_5[0][0]
__________________________________________________________________________________________________
tf.concat (TFOpLambda)          (None, 4)            0           dense_2[0][0]
                                                                 dense_1[0][0]
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 1)            51          dense_6[0][0]
__________________________________________________________________________________________________
dense_8 (Dense)                 (None, 50)           250         tf.split[0][1]
__________________________________________________________________________________________________
tf.concat_2 (TFOpLambda)        (None, 5)            0           tf.concat[0][0]
                                                                 dense_7[0][0]
__________________________________________________________________________________________________
dense_9 (Dense)                 (None, 3)            153         dense_8[0][0]
__________________________________________________________________________________________________
tf.concat_3 (TFOpLambda)        (None, 8)            0           tf.concat_2[0][0]
                                                                 dense_9[0][0]
==================================================================================================
Total params: 102,408
Trainable params: 102,408
Non-trainable params: 0
__________________________________________________________________________________________________

We need to calculate a global rescale factor for the data set based on the size of the images.

# Rescale factor
arg_dict['rend.rescale'] = render.calculate_rescale_factor(dataset)

Next we can render and plot the various S-NeRF outputs : Color, depth map, sun visibility at surface, and albedo. We verify the relighting capability by performing an interpolation between a low and high solar angle. We also generate a flyover video that shows a mixture of relighting and novel view synthesis (can take time to generate).

dataset_rend = render.render_dataset(dataset, model, ['rgb', 'depth', 'ret_sun', 'no_shadow'], arg_dict)

plots.plot_results(dataset['train_imgs'], dataset['train_focals'], dataset_rend['train_rend'])

plots.plot_results(dataset['test_imgs'], dataset['test_focals'], dataset_rend['test_rend'])

plots.render_vertical_depth_comparison(model, arg_dict, dataset['depth_map'])

hwf = [dataset['train_imgs'][0].shape[0], dataset['train_imgs'][0].shape[1], 617000.0/0.3/arg_dict["data.image.df"]]
light_start = np.rad2deg(dataset['train_light_dirs'][4][0,0]), np.rad2deg(dataset['train_light_dirs'][4][0,1])
light_end = np.rad2deg(dataset['train_light_dirs'][11][0,0]), np.rad2deg(dataset['train_light_dirs'][11][0,1])
view_angle=(np.pi, np.pi/2) #Vertical view : elevation = 90 deg, az = 180 deg (north up)

rets=['rgb', 'depth', 'sky', 'no_shadow']
plots.plot_light_angle_inter(model, arg_dict, hwf, light_start, light_end, view_angle, nplots=5, rets=rets)

 outputs = plots.render_flyover_video(f"{arg_dict['out.path']}/", model, arg_dict, hwf, light_start, light_end, rets)

IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (106, 106) to (112, 112) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).
IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (106, 106) to (112, 112) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).
IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (106, 106) to (112, 112) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).
IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (106, 106) to (112, 112) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).

from IPython.display import HTML

HTML("""
<video width="640" height="480" controls>
  <source src="../results/068/flyover_rgb.mp4" type="video/mp4">
</video>
""")