{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# LLM Random Number Picker\n", "\n", "Inspired by this [Information is Beautiful viz on what random number ChatGPT guesses](https://twitter.com/infobeautiful/status/1778059112250589561), I tried the same with 3 LLMs." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import httpx\n", "\n", "\n", "# If we have prior cached data, load it\n", "data = {}\n", "if os.path.exists(\"llm-random-numbers.json\"):\n", " with open(\"llm-random-numbers.json\", \"r\") as f:\n", " data = json.load(f)\n", "\n", "\n", "def claude_haiku(temperature):\n", " r = httpx.post(\n", " \"https://api.anthropic.com/v1/messages\",\n", " headers={\n", " \"x-api-key\": os.environ[\"ANTHROPIC_API_KEY\"],\n", " \"anthropic-version\": \"2023-06-01\",\n", " \"content-type\": \"application/json\",\n", " },\n", " json={\n", " \"model\": \"claude-3-haiku-20240307\",\n", " \"max_tokens\": 2,\n", " \"temperature\": temperature,\n", " \"messages\": [\n", " {\n", " \"role\": \"user\",\n", " \"content\": \"Pick a random number from 0 - 100. Write ONLY the number NOTHING ELSE\",\n", " }\n", " ],\n", " },\n", " )\n", " r.raise_for_status()\n", " return r.json()[\"content\"][0][\"text\"]\n", "\n", "\n", "def openai_gpt_35(temperature):\n", " r = httpx.post(\n", " \"https://api.openai.com/v1/chat/completions\",\n", " headers={\n", " \"Authorization\": f\"Bearer {os.environ['OPENAI_API_KEY']}\",\n", " \"content-type\": \"application/json\",\n", " },\n", " json={\n", " \"model\": \"gpt-3.5-turbo\",\n", " \"max_tokens\": 2,\n", " \"temperature\": temperature,\n", " \"messages\": [\n", " {\n", " \"role\": \"user\",\n", " \"content\": \"Pick a random number from 0 - 100. Write ONLY the number NOTHING ELSE\",\n", " }\n", " ],\n", " },\n", " )\n", " r.raise_for_status()\n", " return r.json()[\"choices\"][0][\"message\"][\"content\"]\n", "\n", "\n", "def google_gemini(temperature):\n", " r = httpx.post(\n", " \"https://europe-west2-aiplatform.googleapis.com/v1/projects/{GOOGLE_PROJECT}/locations/europe-west2/publishers/google/models/gemini-1.0-pro-001:generateContent\".format(**os.environ),\n", " headers={\n", " \"Authorization\": f\"Bearer {os.environ['GOOGLE_API_KEY']}\",\n", " \"content-type\": \"application/json\",\n", " },\n", " json={\n", " \"contents\": [\n", " {\n", " \"role\": \"user\",\n", " \"parts\": [\n", " {\n", " \"text\": \"Pick a random number from 0 - 100. Write ONLY the number NOTHING ELSE\",\n", " }\n", " ]\n", " }\n", " ],\n", " \"generationConfig\": {\n", " \"maxOutputTokens\": 2,\n", " \"temperature\": temperature,\n", " },\n", " },\n", " )\n", " r.raise_for_status()\n", " return r.json()[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n", "\n", "\n", "# Get 200 random numbers\n", "for attempt in range(0, 200):\n", " # At each temperature from 1.0 to 0.0\n", " for t in range(100, -1, -10):\n", " temperature = t / 100\n", " # For each provider\n", " for provider, method in [(\"O\", openai_gpt_35), (\"C\", claude_haiku), (\"G\", google_gemini)]:\n", " # Store data as a key-value pair. The key is \"Provider,Temperature,Attempt\"\n", " key = f\"{provider},{temperature:.2f},{attempt}\"\n", " if key in data:\n", " continue\n", " print(key)\n", " data[key] = method(temperature)\n", " with open(\"llm-random-numbers.json\", \"w\") as f:\n", " json.dump(data, f, sort_keys=True)\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | m | \n", "t | \n", "# | \n", "n | \n", "
---|---|---|---|---|
0 | \n", "C | \n", "0.0 | \n", "0 | \n", "42 | \n", "
1 | \n", "C | \n", "0.0 | \n", "1 | \n", "42 | \n", "
2 | \n", "C | \n", "0.0 | \n", "10 | \n", "42 | \n", "
3 | \n", "C | \n", "0.0 | \n", "100 | \n", "42 | \n", "
4 | \n", "C | \n", "0.0 | \n", "101 | \n", "42 | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
6595 | \n", "O | \n", "1.0 | \n", "95 | \n", "56 | \n", "
6596 | \n", "O | \n", "1.0 | \n", "96 | \n", "74 | \n", "
6597 | \n", "O | \n", "1.0 | \n", "97 | \n", "79 | \n", "
6598 | \n", "O | \n", "1.0 | \n", "98 | \n", "47 | \n", "
6599 | \n", "O | \n", "1.0 | \n", "99 | \n", "67 | \n", "
6600 rows × 4 columns
\n", "