AI powered cross-modal adaptation techniques applied to Sentinel-1 and -2 data

📆 Project Period	November, 2023
📍 GitHub	gitlab.esa.int

Sentinel-1 and Sentinel-2 data fusion through Deep Learning

In this notebook are presented several Deep Learning models to fuse Sentinel-1 and Sentinel-2 data for Land Cover Mapping.

More details can be found in our paper Sebastianelli et al.

To re-train the models set the TRAIN flag to True.

If TRAIN = False pre-trained weights will be loaded.

TRAIN = False

Settings

EPOCHS = 30
BATCH_SIZE = 16

Import libraries

from tensorflow.keras.models import load_model
import matplotlib.pyplot as plt
import numpy as np

Load dataset

The dataset is handeled by the class DatasetHandler.

from DatasetHandler import DatasetHandler
training_handler = DatasetHandler('/home/jovyan/work/S1-S2 Data Fusion/dataset/training')
validation_handler = DatasetHandler('/home/jovyan/work/S1-S2 Data Fusion/dataset/validation')

print('Dataset classes')
for c in training_handler.classes: print('\\t', c)

print('Dataset dimension')
print('\\t', len(training_handler.s2_paths), 'training samples belonging to ', len(training_handler.classes), 'classes')
print('\\t', len(validation_handler.s2_paths), 'validation samples belonging to ', len(validation_handler.classes), 'classes')

Dataset classes
	 /home/jovyan/work/S1-S2 Data Fusion/dataset/training/sentinel2/city
	 /home/jovyan/work/S1-S2 Data Fusion/dataset/training/sentinel2/coastline
	 /home/jovyan/work/S1-S2 Data Fusion/dataset/training/sentinel2/lake
	 /home/jovyan/work/S1-S2 Data Fusion/dataset/training/sentinel2/river
	 /home/jovyan/work/S1-S2 Data Fusion/dataset/training/sentinel2/vegetation
Dataset dimension
	 450 training samples belonging to  5 classes
	 50 validation samples belonging to  5 classes

classes = []
for c in training_handler.classes:
    classes.append(c.split('/')[-1])

Sentinel-2 classifier

from CNN_Classifier import CNN_Classifier
s2classifier = CNN_Classifier((64,64, 12), 5)

/opt/conda/lib/python3.8/site-packages/keras/optimizer_v2/optimizer_v2.py:355: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
  warnings.warn(

if TRAIN == True:
    batch_size = BATCH_SIZE
    training_loader = training_handler.s2_data_loader(batch_size, (64,64,12))
    validation_loader = validation_handler.s2_data_loader(batch_size, (64,64,12))
    training_steps = 4*len(training_handler.s2_paths)
    validation_steps = 4*len(validation_handler.s2_paths)
    epochs = EPOCHS
    #epochs = 5
    s2classifier.train_model(epochs, batch_size, training_loader, validation_loader, training_steps, validation_steps)
    s2classifier.model.save('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-classifier.h5')
else:
    s2classifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-classifier.h5')

Sentinel-1 Classifier

from CNN_Classifier import CNN_Classifier
s1classifier = CNN_Classifier((64,64, 2), 5)

if TRAIN == True:
    training_handler = DatasetHandler('/home/jovyan/work/S1-S2 Data Fusion/dataset/training')
    validation_handler = DatasetHandler('/home/jovyan/work/S1-S2 Data Fusion/dataset/validation')
    batch_size = BATCH_SIZE
    training_loader = training_handler.s1_data_loader(batch_size, (64,64,2))
    validation_loader = validation_handler.s1_data_loader(batch_size, (64,64,2))
    training_steps = 4*len(training_handler.s1_paths)
    validation_steps = 4*len(validation_handler.s1_paths)
    epochs = EPOCHS
    s1classifier.train_model(epochs, batch_size, training_loader, validation_loader, training_steps, validation_steps)
    s1classifier.model.save('/home/jovyan/work/S1-S2 Data Fusion/weights/S1-classifier.h5')
else:
    s1classifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S1-classifier.h5')

Early Fusion

from CNN_Classifier import CNN_Classifier
earlyclassifier = CNN_Classifier((64,64, 12+2), 5)

if TRAIN == True:
    batch_size = BATCH_SIZE
    training_loader = training_handler.s2_s1_data_loader_2(batch_size, (64,64,12), (64,64,2))
    validation_loader = validation_handler.s2_s1_data_loader_2(batch_size, (64,64,12), (64,64,2))
    training_steps = 4*len(training_handler.s2_paths)
    validation_steps = 4*len(validation_handler.s2_paths)
    epochs = EPOCHS
    earlyclassifier.train_model(epochs, batch_size, training_loader, validation_loader, training_steps, validation_steps)
    earlyclassifier.model.save('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-S1-early-classifier.h5')
else:
    earlyclassifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-S1-early-classifier.h5')

Joint Fusion

from Fusion_Classifier import Fusion_Classifier
fclassifier = Fusion_Classifier((64,64, 12), (64,64,2), 5)

if TRAIN == True:
    batch_size = BATCH_SIZE
    training_loader = training_handler.s2_s1_data_loader(batch_size, (64,64,12), (64,64,2))
    validation_loader = validation_handler.s2_s1_data_loader(batch_size, (64,64,12), (64,64,2))
    training_steps = 4*len(training_handler.s1_paths)
    validation_steps = 4*len(validation_handler.s1_paths)
    epochs = EPOCHS
    fclassifier.train_model(epochs, batch_size, training_loader, validation_loader, training_steps, validation_steps)
    fclassifier.model.save('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-S1-classifier.h5')
else:
    fclassifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-S1-classifier.h5')

Late Fusion

s2classifier = CNN_Classifier((64,64, 12), 5)
s1classifier = CNN_Classifier((64,64, 2), 5)
s2classifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S2-classifier.h5')
s1classifier.model = load_model('/home/jovyan/work/S1-S2 Data Fusion/weights/S1-classifier.h5')

validation_loader = validation_handler.s2_s1_data_loader(10*len(validation_handler.s1_paths), (64,64,12), (64,64,2))
s2_s1, g_truth = next(iter(validation_loader))
s2_pre = s2classifier.model.predict(s2_s1[0])
s1_pre = s1classifier.model.predict(s2_s1[1])

Mean Late Fusion

late_sum = []
for i in range(s2_pre.shape[0]):
        late_sum.append(np.argmax((s1_pre[i]+s2_pre[i])))

Weighted Late Fusion

late_weight = []
w1 = np.array([0, 1, 1, 1, 0])
w2 = 1 - w1

for i in range(s2_pre.shape[0]):
        late_weight.append(np.argmax((w1*s1_pre[i]+w2*s2_pre[i])))

Results