{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "2021-07-22-ravelry-pattern-recommender.ipynb",
"provenance": [],
"collapsed_sections": [],
"mount_file_id": "1wWG86wgHlCiQwniLfd6EhuCdMwN0_LMF",
"authorship_tag": "ABX9TyPH10dgglucbENo8EjTWHHo"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "FIOyUcOgRmGf"
},
"source": [
"# Ravelry Pattern Recommender\n",
"> Recommend Wool knitted desings (artifacts) to users\n",
"\n",
"- toc: true\n",
"- badges: true\n",
"- comments: true\n",
"- categories: [SVD, Surprise, API, Art&Culture]\n",
"- image:"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nwRjVKbeKNVT"
},
"source": [
""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7F7Ixp_qLYEG"
},
"source": [
"[Ravelry](https://www.ravelry.com/about) describes itself as a place for knitters, crocheters, designers, spinners, weavers and dyers to keep track of their yarn, tools, project and pattern information, and look to others for ideas and inspiration."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vmY1dm8CORIP"
},
"source": [
"## Setup"
]
},
{
"cell_type": "code",
"metadata": {
"id": "1r1i9yf6Oa7K"
},
"source": [
"!pip install -q surprise"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "Sz5ifkA8OYvb"
},
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import datetime as dt\n",
"\n",
"import surprise\n",
"from surprise.prediction_algorithms import *\n",
"from surprise import Reader, Dataset\n",
"from surprise import SVD, accuracy\n",
"from surprise.model_selection import train_test_split\n",
"from surprise.model_selection import GridSearchCV\n",
"\n",
"import warnings\n",
"warnings.filterwarnings('ignore')"
],
"execution_count": 16,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gXowNBAnmDh9",
"outputId": "b18f0b69-7e47-4302-cb10-c3c86ad26a99"
},
"source": [
"!pip install -q watermark\n",
"%reload_ext watermark\n",
"%watermark -m -iv"
],
"execution_count": 53,
"outputs": [
{
"output_type": "stream",
"text": [
"Compiler : GCC 7.5.0\n",
"OS : Linux\n",
"Release : 5.4.104+\n",
"Machine : x86_64\n",
"Processor : x86_64\n",
"CPU cores : 2\n",
"Architecture: 64bit\n",
"\n",
"numpy : 1.19.5\n",
"IPython : 5.5.0\n",
"pandas : 1.1.5\n",
"surprise: 0.1\n",
"sys : 3.7.11 (default, Jul 3 2021, 18:01:19) \n",
"[GCC 7.5.0]\n",
"\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "d9kuIz7HmDh_"
},
"source": [
"# Poetry is a tool for dependency management and packaging in Python. \n",
"# It allows you to declare the libraries your project depends on and it will manage (install/update) them for you.\n",
"# https://python-poetry.org/docs/basic-usage/\n",
"# !curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -\n",
"# !/root/.local/bin/poetry --version\n",
"# !/root/.local/bin/poetry new poetry-demo\n",
"# %cd poetry-demo\n",
"# !/root/.local/bin/poetry install\n",
"# !/root/.local/bin/poetry add numpy"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "XvWdTkkYQ0NY"
},
"source": [
"## What are patterns?"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "siXUALtfRIJt"
},
"source": [
""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7Ev4xaxlRbnV"
},
"source": [
"> youtube: https://youtu.be/ybEClAPFF8M"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tF7mJenvOU04"
},
"source": [
"## Data"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "UriwzI9lNJVT"
},
"source": [
"### Data Fetching from API (optional)"
]
},
{
"cell_type": "code",
"metadata": {
"id": "_mk9kmfFKbKO"
},
"source": [
"# import pandas as pd\n",
"# import requests\n",
"# import json\n",
"# import random\n",
"# import numpy\n",
"\n",
"# with open('creds.json') as f:\n",
"# creds = json.load(f)\n",
"\n",
"# users = []\n",
"\n",
"# for i in random.sample(range(1, 12000000), 50000):\n",
"# try:\n",
"# url ='https://api.ravelry.com/people/' + str(i) +'.json'\n",
"# response = requests.get(url, auth=(creds['id'], creds['key']))\n",
"# users.append(response.json()['user']['username'])\n",
"# except ValueError: \n",
"# pass\n",
"# if len(set(users)) >10000:\n",
"# break\n",
"\n",
"# parsed_data = []\n",
"\n",
"# for i, user in enumerate(users[9935:]): \n",
" \n",
"# url ='https://api.ravelry.com/projects/' + user + '/list.json?sort=completed_'\n",
"# response = requests.get(url, auth=(creds['id'], creds['key']))\n",
" \n",
"# try:\n",
"# for project in response.json()['projects']:\n",
"# if project['craft_name'] == 'Knitting': \n",
"# if project['pattern_id'] != None:\n",
"# pattern_url ='https://api.ravelry.com/patterns.json?ids=' + str(int(project['pattern_id']))\n",
"# pattern_response = requests.get(pattern_url, auth=(creds['id'], creds['key']))\n",
"# project_tuple = (user, project['completed'], project['rating'], project['status_name'], \n",
"# project['pattern_id'],\n",
"# pattern_response.json()['patterns'][str(int(project['pattern_id']))]['rating_average'],\n",
"# pattern_response.json()['patterns'][str(int(project['pattern_id']))]['rating_count'])\n",
"# parsed_data.append(project_tuple) \n",
" \n",
"# except ValueError:\n",
"# pass\n",
"\n",
"# print(i, len(parsed_data))\n",
"\n",
"# df = pd.DataFrame(parsed_data, columns= ['user', 'completed', 'rating', 'status', 'pattern_id', 'average_rating', 'rating_count'])\n",
"\n",
"# finished_projects = df[df['status'] == 'Finished']\n",
"# finished_projects.to_csv('ravelry_interactions.csv', index=False)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "eW8b8InXSPAq"
},
"source": [
"### Direct loading from drive"
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 419
},
"id": "sTgvoAn-OPY2",
"outputId": "44f81d91-4abb-4849-bc96-48b883d8e468"
},
"source": [
"df = pd.read_csv('https://raw.githubusercontent.com/recohut/reco-data/ravelry/ravelry/v1/ravelry_interactions.csv')\n",
"df"
],
"execution_count": 8,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user | \n",
" completed | \n",
" rating | \n",
" status | \n",
" pattern_id | \n",
" average_rating | \n",
" rating_count | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" hannahcf | \n",
" 2018/11/25 | \n",
" 4.0 | \n",
" Finished | \n",
" 17468 | \n",
" 4.520376 | \n",
" 1276.0 | \n",
"
\n",
" \n",
" | 1 | \n",
" hannahcf | \n",
" 2018/05/05 | \n",
" 4.0 | \n",
" Finished | \n",
" 629964 | \n",
" 4.285714 | \n",
" 7.0 | \n",
"
\n",
" \n",
" | 2 | \n",
" hannahcf | \n",
" 2018/03/13 | \n",
" 4.0 | \n",
" Finished | \n",
" 287992 | \n",
" 4.554957 | \n",
" 928.0 | \n",
"
\n",
" \n",
" | 3 | \n",
" hannahcf | \n",
" 2018/01/31 | \n",
" 4.0 | \n",
" Finished | \n",
" 475167 | \n",
" 4.500000 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 4 | \n",
" hannahcf | \n",
" 2018/01/05 | \n",
" 4.0 | \n",
" Finished | \n",
" 544863 | \n",
" 4.662651 | \n",
" 332.0 | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 11117 | \n",
" creativratte | \n",
" 2015/08/14 | \n",
" 4.0 | \n",
" Finished | \n",
" 223414 | \n",
" 4.611111 | \n",
" 18.0 | \n",
"
\n",
" \n",
" | 11118 | \n",
" creativratte | \n",
" 2015/06/21 | \n",
" 4.0 | \n",
" Finished | \n",
" 467793 | \n",
" 4.667697 | \n",
" 647.0 | \n",
"
\n",
" \n",
" | 11119 | \n",
" creativratte | \n",
" 2015/05/10 | \n",
" 4.0 | \n",
" Finished | \n",
" 211562 | \n",
" 4.668034 | \n",
" 9263.0 | \n",
"
\n",
" \n",
" | 11120 | \n",
" creativratte | \n",
" 2015/02/07 | \n",
" 4.0 | \n",
" Finished | \n",
" 250525 | \n",
" 4.629565 | \n",
" 575.0 | \n",
"
\n",
" \n",
" | 11121 | \n",
" creativratte | \n",
" 2014/09/28 | \n",
" 3.0 | \n",
" Finished | \n",
" 363243 | \n",
" 4.615530 | \n",
" 528.0 | \n",
"
\n",
" \n",
"
\n",
"
11122 rows × 7 columns
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user | \n",
" pattern_id | \n",
" rating | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" hannahcf | \n",
" 17468 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 1 | \n",
" hannahcf | \n",
" 629964 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 2 | \n",
" hannahcf | \n",
" 287992 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 3 | \n",
" hannahcf | \n",
" 475167 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 4 | \n",
" hannahcf | \n",
" 544863 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 11117 | \n",
" creativratte | \n",
" 223414 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11118 | \n",
" creativratte | \n",
" 467793 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11119 | \n",
" creativratte | \n",
" 211562 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11120 | \n",
" creativratte | \n",
" 250525 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11121 | \n",
" creativratte | \n",
" 363243 | \n",
" 3.0 | \n",
"
\n",
" \n",
"
\n",
"
8794 rows × 3 columns
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" count | \n",
" unique | \n",
" top | \n",
" freq | \n",
" mean | \n",
" std | \n",
" min | \n",
" 25% | \n",
" 50% | \n",
" 75% | \n",
" max | \n",
"
\n",
" \n",
" \n",
" \n",
" | user | \n",
" 8794 | \n",
" 383 | \n",
" ciri | \n",
" 348 | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
"
\n",
" \n",
" | pattern_id | \n",
" 8794 | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" 352854 | \n",
" 295926 | \n",
" 16 | \n",
" 106818 | \n",
" 267731 | \n",
" 564484 | \n",
" 1.1567e+06 | \n",
"
\n",
" \n",
" | rating | \n",
" 8794 | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" 3.65783 | \n",
" 0.569459 | \n",
" 0 | \n",
" 3 | \n",
" 4 | \n",
" 4 | \n",
" 4 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user | \n",
" pattern_id | \n",
" rating | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" hannahcf | \n",
" 17468 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 1 | \n",
" hannahcf | \n",
" 629964 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 2 | \n",
" hannahcf | \n",
" 287992 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 3 | \n",
" hannahcf | \n",
" 475167 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 4 | \n",
" hannahcf | \n",
" 544863 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 11117 | \n",
" creativratte | \n",
" 223414 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11118 | \n",
" creativratte | \n",
" 467793 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11119 | \n",
" creativratte | \n",
" 211562 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11120 | \n",
" creativratte | \n",
" 250525 | \n",
" 4.0 | \n",
"
\n",
" \n",
" | 11121 | \n",
" creativratte | \n",
" 363243 | \n",
" 3.0 | \n",
"
\n",
" \n",
"
\n",
"
11122 rows × 3 columns
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" item | \n",
" actual | \n",
" estimated | \n",
"
\n",
" \n",
" \n",
" \n",
" | count | \n",
" 2.000000 | \n",
" 2.000000 | \n",
" 2.000000 | \n",
"
\n",
" \n",
" | mean | \n",
" 269733.500000 | \n",
" 3.500000 | \n",
" 3.798873 | \n",
"
\n",
" \n",
" | std | \n",
" 124075.319788 | \n",
" 0.707107 | \n",
" 0.028703 | \n",
"
\n",
" \n",
" | min | \n",
" 181999.000000 | \n",
" 3.000000 | \n",
" 3.778577 | \n",
"
\n",
" \n",
" | 25% | \n",
" 225866.250000 | \n",
" 3.250000 | \n",
" 3.788725 | \n",
"
\n",
" \n",
" | 50% | \n",
" 269733.500000 | \n",
" 3.500000 | \n",
" 3.798873 | \n",
"
\n",
" \n",
" | 75% | \n",
" 313600.750000 | \n",
" 3.750000 | \n",
" 3.809021 | \n",
"
\n",
" \n",
" | max | \n",
" 357468.000000 | \n",
" 4.000000 | \n",
" 3.819169 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user | \n",
" completed | \n",
" rating | \n",
" status | \n",
" pattern_id | \n",
" average_rating | \n",
" rating_count | \n",
"
\n",
" \n",
" \n",
" \n",
" | 182 | \n",
" Ona | \n",
" 2015/04/22 | \n",
" 3.0 | \n",
" Finished | \n",
" 546649 | \n",
" 4.615385 | \n",
" 13.0 | \n",
"
\n",
" \n",
" | 183 | \n",
" Ona | \n",
" 2015/04/12 | \n",
" 3.0 | \n",
" Finished | \n",
" 393371 | \n",
" 4.531915 | \n",
" 47.0 | \n",
"
\n",
" \n",
" | 184 | \n",
" Ona | \n",
" 2014/11/26 | \n",
" NaN | \n",
" Finished | \n",
" 213483 | \n",
" 4.757460 | \n",
" 1307.0 | \n",
"
\n",
" \n",
" | 185 | \n",
" Ona | \n",
" 2014/08/29 | \n",
" 3.0 | \n",
" Finished | \n",
" 327637 | \n",
" 4.630499 | \n",
" 341.0 | \n",
"
\n",
" \n",
" | 186 | \n",
" Ona | \n",
" 2013/03/20 | \n",
" 4.0 | \n",
" Finished | \n",
" 359828 | \n",
" 4.586093 | \n",
" 302.0 | \n",
"
\n",
" \n",
" | 187 | \n",
" Ona | \n",
" 2013/01/13 | \n",
" 4.0 | \n",
" Finished | \n",
" 89104 | \n",
" 3.933333 | \n",
" 15.0 | \n",
"
\n",
" \n",
" | 188 | \n",
" Ona | \n",
" 2013/01/01 | \n",
" 4.0 | \n",
" Finished | \n",
" 134174 | \n",
" 4.523490 | \n",
" 298.0 | \n",
"
\n",
" \n",
" | 189 | \n",
" Ona | \n",
" 2012/12/27 | \n",
" 4.0 | \n",
" Finished | \n",
" 357468 | \n",
" 4.333333 | \n",
" 60.0 | \n",
"
\n",
" \n",
" | 190 | \n",
" Ona | \n",
" 2012/12/07 | \n",
" 4.0 | \n",
" Finished | \n",
" 10760 | \n",
" 4.209974 | \n",
" 381.0 | \n",
"
\n",
" \n",
" | 191 | \n",
" Ona | \n",
" 2012/12/04 | \n",
" 4.0 | \n",
" Finished | \n",
" 293353 | \n",
" 4.626655 | \n",
" 2341.0 | \n",
"
\n",
" \n",
" | 192 | \n",
" Ona | \n",
" 2012/12/02 | \n",
" 4.0 | \n",
" Finished | \n",
" 357468 | \n",
" 4.333333 | \n",
" 60.0 | \n",
"
\n",
" \n",
" | 193 | \n",
" Ona | \n",
" 2012/11/25 | \n",
" 4.0 | \n",
" Finished | \n",
" 323490 | \n",
" 4.421384 | \n",
" 477.0 | \n",
"
\n",
" \n",
" | 194 | \n",
" Ona | \n",
" 2012/11/12 | \n",
" 4.0 | \n",
" Finished | \n",
" 2525 | \n",
" 4.331984 | \n",
" 494.0 | \n",
"
\n",
" \n",
" | 195 | \n",
" Ona | \n",
" 2012/11/10 | \n",
" 4.0 | \n",
" Finished | \n",
" 88575 | \n",
" 4.441176 | \n",
" 34.0 | \n",
"
\n",
" \n",
" | 196 | \n",
" Ona | \n",
" 2012/10/18 | \n",
" 4.0 | \n",
" Finished | \n",
" 108356 | \n",
" 4.410345 | \n",
" 290.0 | \n",
"
\n",
" \n",
" | 197 | \n",
" Ona | \n",
" 2010/09/03 | \n",
" 3.0 | \n",
" Finished | \n",
" 170995 | \n",
" 4.410042 | \n",
" 239.0 | \n",
"
\n",
" \n",
" | 198 | \n",
" Ona | \n",
" 2010/08/30 | \n",
" 3.0 | \n",
" Finished | \n",
" 181999 | \n",
" 4.400741 | \n",
" 2700.0 | \n",
"
\n",
" \n",
" | 199 | \n",
" Ona | \n",
" 2010/08/27 | \n",
" 3.0 | \n",
" Finished | \n",
" 185648 | \n",
" 4.481057 | \n",
" 1135.0 | \n",
"
\n",
" \n",
" | 200 | \n",
" Ona | \n",
" 2010/08/22 | \n",
" 4.0 | \n",
" Finished | \n",
" 181999 | \n",
" 4.400741 | \n",
" 2700.0 | \n",
"
\n",
" \n",
" | 201 | \n",
" Ona | \n",
" 2010/08/12 | \n",
" 4.0 | \n",
" Finished | \n",
" 156437 | \n",
" 4.608737 | \n",
" 4235.0 | \n",
"
\n",
" \n",
" | 202 | \n",
" Ona | \n",
" 2010/07/26 | \n",
" 4.0 | \n",
" Finished | \n",
" 108356 | \n",
" 4.410345 | \n",
" 290.0 | \n",
"
\n",
" \n",
" | 203 | \n",
" Ona | \n",
" 2010/07/25 | \n",
" 4.0 | \n",
" Finished | \n",
" 92403 | \n",
" 4.309859 | \n",
" 71.0 | \n",
"
\n",
" \n",
" | 204 | \n",
" Ona | \n",
" 2010/07/13 | \n",
" 4.0 | \n",
" Finished | \n",
" 144814 | \n",
" 4.500000 | \n",
" 104.0 | \n",
"
\n",
" \n",
" | 205 | \n",
" Ona | \n",
" 2010/07/06 | \n",
" 4.0 | \n",
" Finished | \n",
" 144814 | \n",
" 4.500000 | \n",
" 104.0 | \n",
"
\n",
" \n",
" | 206 | \n",
" Ona | \n",
" NaN | \n",
" 4.0 | \n",
" Finished | \n",
" 532791 | \n",
" 4.684211 | \n",
" 19.0 | \n",
"
\n",
" \n",
" | 207 | \n",
" Ona | \n",
" NaN | \n",
" NaN | \n",
" Finished | \n",
" 367817 | \n",
" 4.297872 | \n",
" 47.0 | \n",
"
\n",
" \n",
"
\n",
"