import unittest import os import threading from test.support import EnvironmentVarGuard from urllib.parse import urlparse from http.server import BaseHTTPRequestHandler, HTTPServer from google.cloud import bigquery from google.auth.exceptions import DefaultCredentialsError from kaggle_gcp import KaggleKernelCredentials, PublicBigqueryClient class TestBigQuery(unittest.TestCase): def _test_proxy(self, client, should_use_proxy): class HTTPHandler(BaseHTTPRequestHandler): called = False header_found = False def do_HEAD(s): s.send_response(200) def do_GET(s): HTTPHandler.called = True HTTPHandler.header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") s.send_response(200) server_address = urlparse(os.getenv('KAGGLE_DATA_PROXY_URL')) with HTTPServer((server_address.hostname, server_address.port), HTTPHandler) as httpd: threading.Thread(target=httpd.serve_forever).start() try: for ds in client.list_datasets(): pass except: pass httpd.shutdown() if should_use_proxy: self.assertTrue(HTTPHandler.called, msg="Fake server did not receive a request from the BQ client.") self.assertTrue(HTTPHandler.header_found, msg="X-KAGGLE-PROXY-DATA header was missing from the BQ request.") else: self.assertFalse(HTTPHandler.called, msg="Fake server was called from the BQ client, but should not have been.") def test_proxy_using_library(self): env = EnvironmentVarGuard() env.unset('KAGGLE_BQ_USER_JWT') with env: client = PublicBigqueryClient() self._test_proxy(client, should_use_proxy=True) def test_proxy_no_project(self): env = EnvironmentVarGuard() env.unset('KAGGLE_BQ_USER_JWT') with env: client = bigquery.Client() self._test_proxy(client, should_use_proxy=True) def test_project_with_connected_account(self): env = EnvironmentVarGuard() env.set('KAGGLE_BQ_USER_JWT', 'foobar') with env: client = bigquery.Client(project='ANOTHER_PROJECT', credentials=KaggleKernelCredentials()) self._test_proxy(client, should_use_proxy=False) def test_simultaneous_clients(self): env = EnvironmentVarGuard() env.set('KAGGLE_BQ_USER_JWT', 'foobar') with env: proxy_client = bigquery.Client() self._test_proxy(proxy_client, should_use_proxy=True) bq_client = bigquery.Client(project='ANOTHER_PROJECT', credentials=KaggleKernelCredentials()) self._test_proxy(bq_client, should_use_proxy=False) # Verify that proxy client is still going to proxy to ensure global Connection # isn't being modified. self._test_proxy(proxy_client, should_use_proxy=True) def test_no_project_with_connected_account(self): env = EnvironmentVarGuard() env.set('KAGGLE_BQ_USER_JWT', 'foobar') with env: with self.assertRaises(DefaultCredentialsError): # TODO(vimota): Handle this case, either default to Kaggle Proxy or use some default project # by the user or throw a custom exception. client = bigquery.Client(credentials=KaggleKernelCredentials()) self._test_proxy(client, should_use_proxy=False)