AI4EO to monitor air quality(NO2)-omi-linking

📆 Project Period	May, 2022
📍 GitHub	gitlab.esa.int

AI4EO to monitor air quality(NO2)-omi-linking

import numpy as np
import netCDF4
import h5py
import os
import pandas as pd
import glob
from scipy.io import netcdf
from datetime import datetime, timedelta

A. Extract NO2 from OMI-L3

A.1 Explore content of files with h5py

for file in glob.glob('/home/jovyan/eodata/AI4EO to monitor air quality NO2/No2-data/No2-data/OMI-NO2-L3/*.nc4'):
    print(file)
    try:
        root = h5py.File(file, mode='r')
        def print_attrs(name, obj):
            for key, val in obj.attrs.items():
                #print("    %s: %s" % (key, val))
                pass
        root.visititems(print_attrs)

    except OSError as err:
        print(err)
    break #One is enough

A.2 Creating pandas dataframe with date, longitutes, latitude, no2 column data and grid ID

data = []
def yields_data():
    for file in glob.glob('/home/jovyan/eodata/AI4EO to monitor air quality NO2/No2-data/No2-data/OMI-NO2-L3/*.nc4'):
        try:
            root = h5py.File(file, mode='r')
            longitudes = root['XDim'][:]
            latitudes = root['YDim'][:]
            no2 = root['ColumnAmountNO2TropCloudScreened'][:]
            name = os.path.basename(file)
            date = datetime.strptime(str(name[7:11]+"/"+name[12:14]+"/"+name[14:16]), '%Y/%m/%d')
            ref_count = 0
            for lon in range(len(longitudes)):
                for lat in range(len(latitudes)):
                    ref_count += 1
                    yield date, longitudes[lon], latitudes[lat], no2[lat,lon], int(ref_count)
        except Exception as err:
            print(file)
            print(err)

for date, lon, lat, no2, ID in yields_data():
    data.append([date, lon, lat, no2, ID])

df = pd.DataFrame(data, columns=['date', 'longitude', 'latitude', 'no2', 'ID'])
print(df.head)

<bound method NDFrame.head of                date  longitude  latitude           no2    ID
0        2015-05-06     -8.875    49.125 -1.267651e+30     1
1        2015-05-06     -8.875    49.375 -1.267651e+30     2
2        2015-05-06     -8.875    49.625 -1.267651e+30     3
3        2015-05-06     -8.875    49.875 -1.267651e+30     4
4        2015-05-06     -8.875    50.125 -1.267651e+30     5
...             ...        ...       ...           ...   ...
11692027 2009-01-16      1.875    59.875 -1.267651e+30  2108
11692028 2009-01-16      1.875    60.125 -1.267651e+30  2109
11692029 2009-01-16      1.875    60.375 -1.267651e+30  2110
11692030 2009-01-16      1.875    60.625 -1.267651e+30  2111
11692031 2009-01-16      1.875    60.875 -1.267651e+30  2112

[11692032 rows x 5 columns]>

cols = df.columns.values
order = [4,0,2,1,3]

df2 = df[cols[order]].sort_values(['ID', 'date'], ignore_index=True)
df2.to_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/timeSeries_NO2_OMI.csv', index=False)
df2

#print missing dates
all_data = pd.read_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/timeSeries_NO2_OMI.csv')
missings = pd.date_range(start = '2004-10-01', end = '2020-12-31').difference(pd.unique(all_data['date']))
missings

DatetimeIndex(['2004-10-01', '2004-10-02', '2004-10-03', '2004-10-04',
               '2004-10-05', '2004-10-06', '2004-10-07', '2004-10-08',
               '2004-10-09', '2004-10-10',
               ...
               '2020-12-22', '2020-12-23', '2020-12-24', '2020-12-25',
               '2020-12-26', '2020-12-27', '2020-12-28', '2020-12-29',
               '2020-12-30', '2020-12-31'],
              dtype='datetime64[ns]', length=5936, freq=None)

pd.date_range(start = '2004-10-01', end = '2020-12-31').difference(pd.unique(df2['date']))

DatetimeIndex(['2004-11-19', '2004-11-20', '2004-11-21', '2004-11-22',
               '2004-11-23', '2004-11-24', '2004-11-25', '2004-11-26',
               '2004-11-27', '2004-11-28',
               ...
               '2020-12-22', '2020-12-23', '2020-12-24', '2020-12-25',
               '2020-12-26', '2020-12-27', '2020-12-28', '2020-12-29',
               '2020-12-30', '2020-12-31'],
              dtype='datetime64[ns]', length=400, freq=None)

for y in range(2004,2020):
    df2[df2["date"].dt.year==datetime(y,1,1).year].to_csv("/home/jovyan/eodata/AI4EO to monitor air quality NO2/OMI-yearly\\OMI-"+str(y)+".csv", index=False)

B. Link UK grid to OMI grid

B.1 Load UK grid

uk_grids = pd.read_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/UK_grids_noGeo_centroid.csv').drop(['.geo','system:index'], axis=1)
uk_grids = uk_grids.add_prefix('uk_')
uk_grids

B.2 Compute OMI grid values

OMI_all = pd.read_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/timeSeries_NO2_OMI.csv')
OMI_grids = OMI_all.loc[OMI_all['date'] == '2004-10-01']
OMI_grids = OMI_grids.drop(['no2','date'], axis=1)
OMI_grids['latMin'] = OMI_grids['latitude'] - 0.125
OMI_grids['latMax'] = OMI_grids['latitude'] + 0.125
OMI_grids['lonMin'] = OMI_grids['longitude'] - 0.125
OMI_grids['lonMax'] = OMI_grids['longitude'] + 0.125
OMI_grids = OMI_grids.add_prefix('OMI_')
OMI_grids

OMI_grids.to_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/OMI_grids_noGeo_centroid.csv', index=False)

B.3 Link UK grid to OMI grid

for row in uk_grids.head().iterrows():
    #print(row[1])
    print(row[1]['uk_centroidLat'])
    print(row[1]['uk_centroidLon'])

OMI_grids.head()

53.520869128059545
-0.0012246022636999
53.520616071300296
0.0138497457511531
53.52036111946441
0.0289239196618866
53.52010431396655
0.0439978709429132
53.51984553097928
0.0590715504716754

data = []

def closest_OMI_centroid(lat, lon):
    closest_lats = OMI_grids.iloc[(OMI_grids['OMI_latitude']-lat).abs().argsort()]
    clolat = float(closest_lats[:1]['OMI_latitude'])
    closest_lats = closest_lats[closest_lats.OMI_latitude == clolat]
    closest_centroid = closest_lats.iloc[(abs(closest_lats['OMI_longitude'])-abs(lon)).abs().argsort()][:1]
    return closest_centroid

def yields_data():
    for row in uk_grids.iterrows():
        closest = closest_OMI_centroid(row[1]['uk_centroidLat'], row[1]['uk_centroidLon'])
        yield row[1]['uk_gridId'], row[1]['uk_centroidLat'], row[1]['uk_centroidLon'], row[1]['uk_latMax'], \\
                row[1]['uk_latMin'], row[1]['uk_lonMax'], row[1]['uk_lonMin'], int(closest['OMI_ID']), \\
                float(closest['OMI_latitude']), float(closest['OMI_longitude']), float(closest['OMI_latMax']), \\
                float(closest['OMI_latMin']), float(closest['OMI_lonMax']), float(closest['OMI_lonMin'])

for uk_gridId, uk_centroidLat, uk_centroidLon, uk_latMax, uk_latMin, uk_lonMax, uk_lonMin, \\
    OMI_ID, OMI_latitude, OMI_longitude, OMI_latMax, OMI_latMin, OMI_lonMax, OMI_lonMin in yields_data():
    data.append([uk_gridId, uk_centroidLat, uk_centroidLon, uk_latMax, uk_latMin, uk_lonMax, uk_lonMin, \\
                OMI_ID, OMI_latitude, OMI_longitude, OMI_latMax, OMI_latMin, OMI_lonMax, OMI_lonMin])

df = pd.DataFrame(data, columns=['uk_gridId', 'uk_centroidLat', 'uk_centroidLon', 'uk_latMax', 'uk_latMin', \\
                                 'uk_lonMax', 'uk_lonMin', 'OMI_ID', 'OMI_latitude', 'OMI_longitude', \\
                                 'OMI_latMax', 'OMI_latMin', 'OMI_lonMax', 'OMI_lonMin'])

df.to_csv('/home/jovyan/eodata/AI4EO to monitor air quality NO2/link_OMI_UK.csv', index=False)
df

C. Retrieve NO2 time series data for a particular UK grid.

uk_gridIds = df.uk_gridId.unique()
id = uk_gridIds[0]
print('UK grid id:', id)

# Get id of correspondent CAMS grid
cams_gridId = df[df.uk_gridId == id]['OMI_ID'].values[0]
print('Correspondent CAMS grid id:', cams_gridId)

tseries = df2[df2.ID == ID].reset_index()
tseries

UK grid id: TA3204
Correspondent CAMS grid id: 1699