from contextlib import nullcontext import os import sys import tempfile import pytest import setpath # noqa:F401, must come before 'import mechanicalsoup' from bs4 import BeautifulSoup from requests.cookies import RequestsCookieJar from utils import mock_get, prepare_mock_browser import mechanicalsoup def test_submit_online(httpbin): """Complete and submit the pizza form at http://httpbin.org/forms/post """ browser = mechanicalsoup.Browser() page = browser.get(httpbin + "/forms/post") form = page.soup.form form.find("input", {"name": "custname"})["value"] = "Philip J. Fry" # leave custtel blank without value assert "value" not in form.find("input", {"name": "custtel"}).attrs form.find("input", {"name": "size", "value": "medium"})["checked"] = "" form.find("input", {"name": "topping", "value": "cheese"})["checked"] = "" form.find("input", {"name": "topping", "value": "onion"})["checked"] = "" form.find("textarea", {"name": "comments"}).insert(0, "freezer") response = browser.submit(form, page.url) # helpfully the form submits to http://httpbin.org/post which simply # returns the request headers in json format json = response.json() data = json["form"] assert data["custname"] == "Philip J. Fry" assert data["custtel"] == "" # web browser submits "" for input left blank assert data["size"] == "medium" assert data["topping"] == ["cheese", "onion"] assert data["comments"] == "freezer" assert json["headers"]["User-Agent"].startswith('python-requests/') assert 'MechanicalSoup' in json["headers"]["User-Agent"] def test_get_request_kwargs(httpbin): """Return kwargs without a submit""" browser = mechanicalsoup.Browser() page = browser.get(httpbin + "/forms/post") form = page.soup.form form.find("input", {"name": "custname"})["value"] = "Philip J. Fry" request_kwargs = browser.get_request_kwargs(form, page.url) assert "method" in request_kwargs assert "url" in request_kwargs assert "data" in request_kwargs assert ("custname", "Philip J. Fry") in request_kwargs["data"] def test_get_request_kwargs_when_method_is_in_kwargs(httpbin): """Raise TypeError exception""" browser = mechanicalsoup.Browser() page = browser.get(httpbin + "/forms/post") form = page.soup.form kwargs = {"method": "post"} with pytest.raises(TypeError): browser.get_request_kwargs(form, page.url, **kwargs) def test_get_request_kwargs_when_url_is_in_kwargs(httpbin): """Raise TypeError exception""" browser = mechanicalsoup.Browser() page = browser.get(httpbin + "/forms/post") form = page.soup.form kwargs = {"url": httpbin + "/forms/post"} with pytest.raises(TypeError): # pylint: disable=redundant-keyword-arg browser.get_request_kwargs(form, page.url, **kwargs) def test_get_request_kwargs_when_submit_element_has_formaction_attribute(): form_html = """
""" form = BeautifulSoup(form_html, "lxml").form request_kwargs = mechanicalsoup.Browser.get_request_kwargs( form, "https://example.com" ) assert request_kwargs["url"] == "https://example.com/submit1" def test_get_request_kwargs_many_submit_elements_have_formaction_attribute(): form_html = """
""" browser = mechanicalsoup.StatefulBrowser() browser.open_fake_page(form_html, url="https://example.com") browser.select_form() chosen_button = browser.form.form.find_all( "button" )[1] # Choose the second button browser.form.choose_submit( submit=chosen_button ) assert len(browser.form.form.find_all("button")) == 1 request_kwargs = mechanicalsoup.Browser.get_request_kwargs( browser.form.form, "https://example.com" ) assert request_kwargs["url"] == "https://example.com/submit2" def test__request(httpbin): form_html = f"""
Pizza Size

Small

Medium

Large

Pizza Toppings

Bacon

Extra Cheese

Onion

Mushroom

""" form = BeautifulSoup(form_html, "lxml").form browser = mechanicalsoup.Browser() response = browser._request(form) data = response.json()['form'] assert data["customer"] == "Philip J. Fry" assert data["telephone"] == "555" assert data["comments"] == "freezer" assert data["size"] == "medium" assert data["topping"] == ["bacon", "onion"] assert data["shape"] == "square" assert "application/x-www-form-urlencoded" in response.request.headers[ "Content-Type"] valid_enctypes_file_submit = {"multipart/form-data": True, "application/x-www-form-urlencoded": False } default_enctype = "application/x-www-form-urlencoded" @pytest.mark.parametrize("file_field", [ """""", ""]) @pytest.mark.parametrize("submit_file", [ True, False ]) @pytest.mark.parametrize("enctype", [ pytest.param("multipart/form-data"), pytest.param("application/x-www-form-urlencoded"), pytest.param("Invalid enctype") ]) def test_enctype_and_file_submit(httpbin, enctype, submit_file, file_field): # test if enctype is respected when specified # and if files are processed correctly form_html = f"""
{file_field}
""" form = BeautifulSoup(form_html, "lxml").form valid_enctype = (enctype in valid_enctypes_file_submit and valid_enctypes_file_submit[enctype]) if submit_file and file_field: # create a temporary file for testing file upload file_content = b":-)" pic_filedescriptor, pic_path = tempfile.mkstemp() pic_filename = os.path.basename(pic_path) os.write(pic_filedescriptor, file_content) os.close(pic_filedescriptor) if valid_enctype: # Correct encoding => send the content expected_content = file_content else: # Encoding doesn't allow sending the content, we expect # the filename as a normal text field. expected_content = os.path.basename(pic_path.encode()) else: pic_path = None expected_content = b"" # update the form (if needed) and submit browser = mechanicalsoup.Browser() with open(pic_path, "rb") if pic_path is not None else nullcontext() as fd: if pic_path is not None: tag = form.find("input", {"name": "pic"}) tag["value"] = fd response = browser._request(form) if enctype not in valid_enctypes_file_submit: expected_enctype = default_enctype else: expected_enctype = enctype assert expected_enctype in response.request.headers["Content-Type"] resp = response.json() assert resp["form"]["in"] == "test" found = False found_in = None for key, value in resp.items(): if value: if "pic" in value: content = value["pic"].encode() assert not found assert key in ("files", "form") found = True found_in = key if key == "files" and not valid_enctype: assert not value assert found == bool(file_field) if file_field: assert content == expected_content if valid_enctype: assert found_in == "files" if submit_file: assert ("filename=\"" + pic_filename + "\"" ).encode() in response.request.body else: assert b"filename=\"\"" in response.request.body else: assert found_in == "form" if submit_file and file_field: os.remove(pic_path) def test__request_select_none(httpbin): """Make sure that a """ form = BeautifulSoup(form_html, "lxml").form browser = mechanicalsoup.Browser() response = browser._request(form) assert response.json()['form'] == {'shape': 'round'} def test__request_disabled_attr(httpbin): """Make sure that disabled form controls are not submitted.""" form_html = f"""
""" browser = mechanicalsoup.Browser() response = browser._request(BeautifulSoup(form_html, "lxml").form) assert response.json()['form'] == {} @pytest.mark.parametrize("keyword", [ pytest.param('method'), pytest.param('url'), ]) def test_request_keyword_error(keyword): """Make sure exception is raised if kwargs duplicates an arg.""" form_html = "
" browser = mechanicalsoup.Browser() with pytest.raises(TypeError, match="multiple values for"): browser._request(BeautifulSoup(form_html, "lxml").form, 'myurl', **{keyword: 'somevalue'}) def test_no_404(httpbin): browser = mechanicalsoup.Browser() resp = browser.get(httpbin + "/nosuchpage") assert resp.status_code == 404 def test_404(httpbin): browser = mechanicalsoup.Browser(raise_on_404=True) with pytest.raises(mechanicalsoup.LinkNotFoundError): browser.get(httpbin + "/nosuchpage") resp = browser.get(httpbin.url) assert resp.status_code == 200 def test_set_cookiejar(httpbin): """Set cookies locally and test that they are received remotely.""" # construct a phony cookiejar and attach it to the session jar = RequestsCookieJar() jar.set('field', 'value') assert jar.get('field') == 'value' browser = mechanicalsoup.Browser() browser.set_cookiejar(jar) resp = browser.get(httpbin + "/cookies") assert resp.json() == {'cookies': {'field': 'value'}} def test_get_cookiejar(httpbin): """Test that cookies set by the remote host update our session.""" browser = mechanicalsoup.Browser() resp = browser.get(httpbin + "/cookies/set?k1=v1&k2=v2") assert resp.json() == {'cookies': {'k1': 'v1', 'k2': 'v2'}} jar = browser.get_cookiejar() assert jar.get('k1') == 'v1' assert jar.get('k2') == 'v2' def test_post(httpbin): browser = mechanicalsoup.Browser() data = {'color': 'blue', 'colorblind': 'True'} resp = browser.post(httpbin + "/post", data) assert resp.status_code == 200 and resp.json()['form'] == data def test_put(httpbin): browser = mechanicalsoup.Browser() data = {'color': 'blue', 'colorblind': 'True'} resp = browser.put(httpbin + "/put", data) assert resp.status_code == 200 and resp.json()['form'] == data @pytest.mark.parametrize("http_html_expected_encoding", [ pytest.param((None, 'utf-8', 'utf-8')), pytest.param(('utf-8', 'utf-8', 'utf-8')), pytest.param(('utf-8', None, 'utf-8')), pytest.param(('utf-8', 'ISO-8859-1', 'utf-8')), ]) def test_encoding(httpbin, http_html_expected_encoding): http_encoding = http_html_expected_encoding[0] html_encoding = http_html_expected_encoding[1] expected_encoding = http_html_expected_encoding[2] url = 'mock://encoding' text = ( '' + '' + ( ( 'Titleéàè' ) if html_encoding else '' ) + '' + '' ) browser, adapter = prepare_mock_browser() mock_get( adapter, url=url, reply=( text.encode(http_encoding) if http_encoding else text.encode("utf-8") ), content_type=( 'text/html' + ( ';charset=' + http_encoding if http_encoding else '' ) ) ) browser.open(url) assert browser.page.original_encoding == expected_encoding if __name__ == '__main__': pytest.main(sys.argv)