#!/bin/bash # Matheus Fillipe --- 25/01/2022 # wcofun.com scraper MIT ########################################################################################## # Things you might want to change: # MENU_CMD. Choose your menu. # NOTIFY_CMD. How to tell you about the status. Leave empty for a simple echo to stdout if [ -t 0 ] # if on a terminal then MENU_CMD="fzf" NOTIFY_CMD="" else MENU_CMD="rofi -dmenu -i" # MENU_CMD="dmenu" NOTIFY_CMD="notify-send WCOFUN" fi # User agent. Maybe you have problems with this UA="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" # Use system's curl if curl-impersonate is not found for curl in \ curl-impersonate \ curl-impersonate-chrome \ curl-impersonate-ff \ curl-impersonate-firefox \ curl do hash "$curl" &>/dev/null && break done CURL_PATH="$curl" # Maybe you want to use a proxy? (Helps when getting blocked by cloudflare, or if you want to debug) CURL_EXTRA_PARAMS="" # CURL_EXTRA_PARAMS="-k --tlsv1 -x http://localhost:8080" PRE_COMMAND="" # PRE_COMMAND="mitmdump 2>&1 /dev/null" CONFIG_FILE=~/.wcofunrc ########################################################################################## # Colors RED='\033[0;31m' GREEN='\033[0;32m' WHITE='\033[1;37m' BLUE='\033[0;34m' BROWN='\033[0;33m' PURPLE='\033[0;35m' NC='\033[0m' red()(set -o pipefail;"$@" 2>&1>&3|sed $'s,.*,\e[31m&\e[m,'>&2)3>&1 # if the config file exists, it will override the variables above if [ -f ~/.wcofunrc ] then echo "Loading config file ~/.wcofunrc" source "$CONFIG_FILE" fi if [ -z "$CURL_PATH" ] then echo -e "${RED}Curl executable not found. You need to set CURL_PATH or install curl{$NC}" exit 1 fi baseurl="https://www.wcofun.net/" domainname="www.wcofun.net" storage="$HOME/.local/share/wcofun.cli/" get_cache () { if [ ! -f "$1" ]; then echo '{}' > "$1" fi cat "$1" } get_cache_search () { get_cache "$storage/search.json" } get_cache_episode () { get_cache "$storage/episode.json" } get_cache_series_url () { get_cache "$storage/series_url.json" } save_cache() { echo "$1" > "$2" } save_cache_search () { save_cache "$1" "$storage/search.json" } save_cache_episode () { save_cache "$1" "$storage/episode.json" } save_cache_series_url () { save_cache "$1" "$storage/series_url.json" } finalize (){ [ -n "$PRE_COMMAND" ] && kill %1 exit 0 } mkdir -p "$storage" touch "$storage/lastq" trap finalize INT menu_cmd () { case "$MENU_CMD" in "fzf"*) args=$(echo "$*" | sed -re 's/""(.+)""/"\1"/g') eval "$MENU_CMD $args" ;; "rofi"*) args=${*//--prompt/-p} args=${args//--query/-filter} args=$(echo "$args" | sed -re 's/""(.+)""/"\1"/g') eval "$MENU_CMD $args" ;; "dmenu"*) args=${*//--prompt/-p} args=${args//--print-query/} args=${args//--query \"*\"/} args=$(echo "$args" | sed -re 's/""(.+)""/"\1"/g') eval "$MENU_CMD $args" ;; *) echo "Warning: Unknown menu command: $MENU_CMD" eval "$MENU_CMD $*" ;; esac } notify_cmd () { echo "$*" if [ -n "$NOTIFY_CMD" ]; then $NOTIFY_CMD "$1" fi } chru() { printf "\x$(printf %x "$1")"; } get_link() { link=$1 script=$( "$CURL_PATH" --silent $CURL_EXTRA_PARAMS "$link" -H "$UA" \ -H "authority: $domainname" \ -H 'pragma: no-cache' \ -H 'cache-control: no-cache' \ -H 'accept: application/json, text/javascript, */*; q=0.01' \ -H "$UA" \ -H 'x-requested-with: XMLHttpRequest' \ -H 'sec-gpc: 1' \ -H 'sec-fetch-site: same-origin' \ -H 'sec-fetch-mode: cors' \ -H 'sec-fetch-dest: empty' \ -H "Referer: $baseurl" \ -H 'accept-language: en-US,en;q=0.9' \ --compressed | htmlq 'div.fourteen.columns script:nth-child(2)' | sed 's/