#!/usr/bin/env python3
from atsd_client import connect
from atsd_client.models import *
from atsd_client.services import *
from prettytable import PrettyTable

import logging
logger = logging.getLogger()
logger.disabled = True

# Connect to ATSD server
connection = connect('connection.properties')
#connection = connect_url('https://atsd_hostname:8443', 'user', 'password')

# specify date interval
startDate = "previous_day"
endDate =   "current_day"
tags = {'http_referer': '*'}

service = MessageService(connection)

ef = EntityFilter(entity="*")
df = DateFilter(start_date=startDate, end_date=endDate)
query = MessageQuery(entity_filter=ef, date_filter=df, type="web", source="access.log", tags=tags, limit=10000)
message_list = service.query(query)
t = PrettyTable(['Date', 'URI', 'Referrer', 'IP', 'Org'])

excluded_domains = ['google', 'axibase.com', 'yandex.ru', 'baidu.com', 'statcounter.com', 'bing.com', 'duckduckgo.com', 'jshell.net', 'facebook.com']

for msg in message_list:
    rf = msg.tags['http_referer']
    uri = msg.tags['request_uri']

    if any(domain in rf for domain in excluded_domains):
        continue
    if '/ec2-monitoring' in uri:
        continue
    t.add_row([msg.date.strftime("%Y-%m-%d %H:%M"), uri, rf, msg.tags['remote_addr'], msg.tags['geoip_org']])

t.sortby = "Date"

print(t.get_html_string(title="Daily Referrer Requests"))