import os, io, subprocess import pandas as pd import csv, time if os.environ.get('DISPLAY') is None: # this variable will not be set when X Server is not running import matplotlib as mplib # set a different display back-end to squash the error mplib.use('Agg') import matplotlib.pyplot as plt import seaborn as sns # define parser to convert unix timestamp to datetime def date_parser(string_list): return [time.ctime(float(x)) for x in string_list] # Set output directory. This should be in the dump1090 html directory for easy access via browser. Needs permissions set to enable writing, or run the script with sudo. outdir = '/usr/share/skyaware/html/plots/' # get location of rrd data from graphs1090 config with open("/etc/default/graphs1090") as file: for line in file: if line.startswith("DB="): rrdloc = line.replace("DB=",'').rstrip() break # test if UAT data is present if os.path.isfile(os.path.join(rrdloc,"localhost/dump1090-localhost/dump1090_aircraft-recent_978.rrd")): print("UAT") UAT = True else: print("No UAT") UAT = False # get data from rrd databases print("Getting data...") rrdremote = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_messages-remote_accepted.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) remote_messages = io.StringIO(rrdremote.stdout.replace(':','')) rrdlocal = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_messages-local_accepted.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) local_messages = io.StringIO(rrdlocal.stdout.replace(':','')) rrdrange = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_range-max_range.rrd MAX -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) range = io.StringIO(rrdrange.stdout.replace(':','')) rrdaircraft = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_aircraft-recent.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) aircraft = io.StringIO(rrdaircraft.stdout.replace(':','')) rrdmlat = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_mlat-recent.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) mlat = io.StringIO(rrdmlat.stdout.replace(':','')) if UAT: uatrrdmessages = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_messages-messages_978.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) uatmessages = io.StringIO(uatrrdmessages.stdout.replace(':','')) uatrrdaircraft = subprocess.run(['rrdtool fetch ./localhost/dump1090-localhost/dump1090_aircraft-recent_978.rrd AVERAGE -s "end-30d-23h45min" -e "23:30 yesterday" -a'], cwd=rrdloc, capture_output=True, text=True, shell=True) uataircraft = io.StringIO(uatrrdaircraft.stdout.replace(':','')) print("Processing...") # import rrd data to dataframes df_remote = pd.read_csv(remote_messages, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'remote'], header=None, skiprows=2) df_local = pd.read_csv(local_messages, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'local'], header=None, skiprows=2) df_range = pd.read_csv(range, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'range'], header=None, skiprows=2) df_aircraft = pd.read_csv(aircraft, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'total', 'positions'], header=None, skiprows=2) df_mlat = pd.read_csv(mlat, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'mlat'], header=None, skiprows=2) if UAT: df_uatmessages = pd.read_csv(uatmessages, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'messages'], header=None, skiprows=2) df_uataircraft = pd.read_csv(uataircraft, parse_dates=[0], sep='\s+', date_parser=date_parser, index_col='DateTime', names=['DateTime', 'total', 'positions'], header=None, skiprows=2) # sum remote and local messages df_messages = pd.merge_asof(df_remote, df_local, on='DateTime') df_messages = df_messages.set_index('DateTime') df_messages['messages'] = df_remote['remote'] + df_local['local'] # calculate range in nm and add column to dataframe df_range['rangenm'] = df_range['range'] / 1852 # get range stats to set colour bar range_min = df_range['rangenm'].min() range_max = df_range['rangenm'].max() range_mean = df_range['rangenm'].mean() range_median = df_range['rangenm'].median() # replace NaN with zeros so seaborn doesn't complain df_messages = df_messages.fillna(0) df_range = df_range.fillna(0) df_aircraft = df_aircraft.fillna(0) df_mlat = df_mlat.fillna(0) if UAT: df_uatmessages = df_uatmessages.fillna(0) df_uataircraft = df_uataircraft.fillna(0) # add date and time columns from index df_messages['date'] = df_messages.index.date df_messages['time'] = df_messages.index.time df_range['date'] = df_range.index.date df_range['time'] = df_range.index.time df_aircraft['date'] = df_aircraft.index.date df_aircraft['time'] = df_aircraft.index.time df_mlat['date'] = df_mlat.index.date df_mlat['time'] = df_mlat.index.time if UAT: df_uatmessages['date'] = df_uatmessages.index.date df_uatmessages['time'] = df_uatmessages.index.time df_uataircraft['date'] = df_uataircraft.index.date df_uataircraft['time'] = df_uataircraft.index.time # convert dataframe to wideformat for plotting messages_w = df_messages.pivot(index='date', columns='time', values='messages') range_w = df_range.pivot(index='date', columns='time', values='rangenm') aircraft_w = df_aircraft.pivot(index='date', columns='time', values='total') mlat_w = df_mlat.pivot(index='date', columns='time', values='mlat') if UAT: uatmessages_w = df_uatmessages.pivot(index='date', columns='time', values='messages') uataircraft_w = df_uataircraft.pivot(index='date', columns='time', values='total') print("Generating Plots...") # generate plots fig, ax = plt.subplots(figsize=(30,12), dpi=80) sns.heatmap(messages_w, cmap="magma", cbar_kws={"shrink": 0.5}) filename = 'messages.png' plt.title("Messages per second") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir, filename), bbox_inches="tight") plt.clf() filename = 'range.png' sns.heatmap(range_w, vmin=range_min, vmax=range_max, cmap='seismic', center=range_mean, cbar_kws={"shrink":0.5}) plt.title("Maximum Range") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir, filename), bbox_inches="tight") plt.clf() filename = 'aircraft.png' sns.heatmap(aircraft_w, cmap="magma", cbar_kws={"shrink": 0.5}) plt.title("Total Aircraft") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir,filename), bbox_inches="tight") plt.clf() filename = 'mlat.png' sns.heatmap(mlat_w, cmap='magma', cbar_kws={"shrink": 0.5}) plt.title("MLAT Aircraft") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir, filename), bbox_inches="tight") plt.clf() if UAT: filename = 'uatmessages.png' sns.heatmap(uatmessages_w, cmap='magma', cbar_kws={"shrink": 0.5}) plt.title("UAT-978 Messages") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir, filename), bbox_inches="tight") plt.clf() filename = "uataircraft.png" sns.heatmap(uataircraft_w, cmap='magma', cbar_kws={"shrink": 0.5}) plt.title("UAT-978 Aircraft") plt.xlabel("Time") plt.ylabel("Date") plt.savefig(os.path.join(outdir, filename), bbox_inches="tight") # write basic webpage for easy access to plots. htmlstart = """

Month Heatmaps

Data taken from Graphs1090 with 15 minute resolution. Messages and Aircraft plots are 15 minute averages, Range is the 15 minute peak.

Messages All aircraft MLAT Aircraft """ htmluat = """Messages All aircraft """ htmlend = """Range """ if UAT: html = htmlstart + htmluat + htmlend else: html = htmlstart + htmlend file = open(os.path.join(outdir,"heatmaps.html"), "w") file.write(html) file.close() print("Plots can be found in " + outdir + " or via browser at /skyaware/plots/heatmaps.html")