// ==UserScript==
// @name Export Twitter Following List
// @namespace https://github.com/prinsss/
// @version 1.0.0
// @description Export your Twitter/X's following/followers list to a CSV/JSON/HTML file.
// @author prin
// @match *://twitter.com/*
// @icon https://www.google.com/s2/favicons?sz=64&domain=twitter.com
// @grant unsafeWindow
// @run-at document-start
// @supportURL https://github.com/prinsss/export-twitter-following-list/issues
// @updateURL https://raw.githubusercontent.com/prinsss/export-twitter-following-list/master/export-twitter-following-list.user.js
// @downloadURL https://raw.githubusercontent.com/prinsss/export-twitter-following-list/master/export-twitter-following-list.user.js
// @license MIT
// ==/UserScript==
(function () {
'use strict';
/*
|--------------------------------------------------------------------------
| Global Variables
|--------------------------------------------------------------------------
*/
const SCRIPT_NAME = 'export-twitter-following-list';
/** @type {Element} */
let panelDom = null;
/** @type {Element} */
let listContainerDom = null;
/** @type {IDBDatabase} */
let db = null;
let isList = false;
let savedCount = 0;
let targetUser = '';
let currentType = '';
let previousPathname = '';
const infoLogs = [];
const errorLogs = [];
const buffer = new Set();
const currentList = new Map();
const currentListSwapped = new Map();
const currentListUniqueSet = new Set();
/*
|--------------------------------------------------------------------------
| Script Bootstraper
|--------------------------------------------------------------------------
*/
initDatabase();
hookIntoXHR();
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', onPageLoaded);
} else {
onPageLoaded();
}
// Determine wether the script should be run.
function bootstrap() {
const pathname = location.pathname;
if (pathname === previousPathname) {
return;
}
previousPathname = pathname;
// Show the script UI on these pages:
// - User's following list
// - User's followers list
// - List's member list
// - List's followers list
const listRegex = /^\/i\/lists\/(.+)\/(followers|members)/;
const userRegex = /^\/(.+)\/(following|followers_you_follow|followers|verified_followers)/;
isList = listRegex.test(pathname);
const isUser = userRegex.test(pathname);
if (!isList && !isUser) {
destroyControlPanel();
return;
}
const regex = isList ? listRegex : userRegex;
const parsed = regex.exec(pathname) || [];
const [match, target, type] = parsed;
initControlPanel();
updateControlPanel({ type, username: isList ? `list_${target}` : target });
}
// Listen to URL changes.
function onPageLoaded() {
new MutationObserver(bootstrap).observe(document.head, {
childList: true,
});
info('Script ready.');
}
/*
|--------------------------------------------------------------------------
| Page Scroll Listener
|--------------------------------------------------------------------------
*/
// When the content of the list changes, we extract some information from the DOM.
// Note that Twitter is using Virtual List so DOM nodes are always recycled.
function onListChange() {
listContainerDom.childNodes.forEach((child) => {
// NOTE: This may vary as Twitter upgrades.
const link = child.querySelector(
'div[role=button] > div:first-child > div:nth-child(2) > div:first-child ' +
'> div:first-child > div:first-child > div:nth-child(2) a'
);
if (!link) {
debug('No link element found in list child', child);
return;
}
const span = link.querySelector('span');
const parsed = /@(\w+)/.exec(span.textContent) || [];
const [match, username] = parsed;
if (!username) {
debug('No username found in the link', span.textContent, child);
return;
}
// We use a emoji to mark that a user was added into current exporting list.
const mark = ' ✅';
if (currentListUniqueSet.has(username)) {
// When you scroll back, the DOM was reset so we need to mark it again.
if (!span.textContent.includes(mark)) {
const index = currentListSwapped.get(username);
span.innerHTML += `${mark}😸 (${index})`;
}
return;
}
savedCount += 1;
updateControlPanel({ count: savedCount });
// Add the username extracted to the exporting list.
const index = savedCount;
currentListUniqueSet.add(username);
currentList.set(index, username);
currentListSwapped.set(username, index);
span.innerHTML += `${mark} (${index})`;
});
}
function attachToListContainer() {
// NOTE: This may vary as Twitter upgrades.
if (isList) {
listContainerDom = document.querySelector(
'div[role="group"] div[role="dialog"] section[role="region"] > div > div'
);
} else {
listContainerDom = document.querySelector(
'main[role="main"] div[data-testid="primaryColumn"] section[role="region"] > div > div'
);
}
if (!listContainerDom) {
error(
'No list container found. ' +
'This may be a problem caused by Twitter updates. Please file an issue on GitHub: ' +
'https://github.com/prinsss/export-twitter-following-list/issues'
);
return;
}
// Add a border to the attached list container as an indicator.
listContainerDom.style.border = '2px dashed #1d9bf0';
// Listen to the change of the list.
onListChange();
new MutationObserver(onListChange).observe(listContainerDom, {
childList: true,
});
}
/*
|--------------------------------------------------------------------------
| User Interfaces
|--------------------------------------------------------------------------
*/
// Hide the script UI and clear all the cache.
function destroyControlPanel() {
document.getElementById(`${SCRIPT_NAME}-panel`)?.remove();
document.getElementById(`${SCRIPT_NAME}-panel-style`)?.remove();
panelDom = null;
listContainerDom = null;
currentType = '';
targetUser = '';
savedCount = 0;
currentList.clear();
currentListUniqueSet.clear();
currentListSwapped.clear();
}
// Update the script UI.
function updateControlPanel({ type, username, count = 0 }) {
if (!panelDom) {
error('Monitor panel is not initialized');
return;
}
if (type) {
currentType = type;
panelDom.querySelector('#list-type').textContent = type;
}
if (count) {
panelDom.querySelector('#saved-count').textContent = count;
}
if (username) {
targetUser = username;
panelDom.querySelector('#target-user').textContent = username;
}
}
// Show the script UI.
function initControlPanel() {
destroyControlPanel();
const panel = document.createElement('div');
panelDom = panel;
panel.id = `${SCRIPT_NAME}-panel`;
panel.innerHTML = `
`;
tableBody.appendChild(column);
});
return table.outerHTML;
}
/*
|--------------------------------------------------------------------------
| Button Events
|--------------------------------------------------------------------------
*/
function onExportStart() {
info('Start listening on page scroll...');
attachToListContainer();
info('Scroll down the page and the list content will be saved automatically as you scroll.');
info('Tips: Do not scroll too fast since the list is lazy-loaded.');
}
function onExportDismiss() {
destroyControlPanel();
}
function onExportPreview() {
openPreviewModal();
}
async function onExportCSV() {
try {
const filename = `twitter-${targetUser}-${currentType}-${Date.now()}.csv`;
info('Exporting to CSV file: ' + filename);
const content = await exportToCSVFormat();
saveFile(filename, content);
} catch (err) {
error(err.message, err);
}
}
async function onExportJSON() {
try {
const filename = `twitter-${targetUser}-${currentType}-${Date.now()}.json`;
info('Exporting to JSON file: ' + filename);
const content = await exportToJSONFormat();
saveFile(filename, content);
} catch (err) {
error(err.message, err);
}
}
async function onExportHTML() {
try {
const filename = `twitter-${targetUser}-${currentType}-${Date.now()}.html`;
info('Exporting to HTML file: ' + filename);
const content = await exportToHTMLFormat();
saveFile(filename, content);
} catch (err) {
error(err.message, err);
}
}
async function onDumpDatabase() {
try {
const filename = `${SCRIPT_NAME}-database-dump-${Date.now()}.json`;
info('Exporting IndexedDB to file: ' + filename);
const obj = await dumpDatabase();
saveFile(filename, JSON.stringify(obj, undefined, ' '));
} catch (err) {
error(err.message, err);
}
}
/*
|--------------------------------------------------------------------------
| Database Management
|--------------------------------------------------------------------------
*/
function initDatabase() {
const request = indexedDB.open(SCRIPT_NAME, 1);
request.onerror = (event) => {
error('Failed to open database.', event);
};
request.onsuccess = () => {
db = request.result;
info('New connection to IndexedDB opened.');
// Flush buffer if there is any incoming data received before the DB is ready.
if (buffer.size) {
insertUserDataIntoDatabase([]);
}
};
request.onupgradeneeded = (event) => {
db = event.target.result;
info('New IndexedDB initialized.');
// Use the numeric user ID as primary key and the username as index for lookup.
const objectStore = db.createObjectStore('users', { keyPath: 'rest_id' });
objectStore.createIndex('screen_name', 'legacy.screen_name', { unique: false });
if (buffer.size) {
insertUserDataIntoDatabase([]);
}
};
}
function insertUserDataIntoDatabase(users) {
// Add incoming data to a buffer queue.
users.forEach((user) => buffer.add(user));
// If the DB is not ready yet at this point, queue the data and wait for it.
if (!db) {
info(`Added ${users.length} users to buffer`);
if (buffer.size > 100) {
error('The database is not initialized.');
error('Maximum buffer size exceeded. Current: ' + buffer.size);
}
return;
}
const toBeInserted = [...buffer.values()];
const insertLength = toBeInserted.length;
const transaction = db.transaction('users', 'readwrite');
const objectStore = transaction.objectStore('users');
transaction.oncomplete = () => {
info(`Added ${insertLength} users to database.`);
for (const item of toBeInserted) {
buffer.delete(item);
}
};
transaction.onerror = (event) => {
error(`Failed to add ${insertLength} users to database.`, event);
};
// Insert or update the user data.
toBeInserted.forEach((user) => {
const request = objectStore.put(user);
request.onerror = function (event) {
error(`Failed to write database. User ID: ${user.id}`, event, user);
};
});
}
// Get a user's record from database by his username.
async function queryDatabaseByUsername(username) {
if (!db) {
error('The database is not initialized.');
return;
}
const transaction = db.transaction('users', 'readonly');
const objectStore = transaction.objectStore('users');
// Use the defined index to look up.
const index = objectStore.index('screen_name');
const request = index.get(username);
return new Promise((resolve) => {
request.onsuccess = () => {
resolve(request.result);
};
request.onerror = (event) => {
error(`Failed to query user ${username} from database.`, event);
resolve(null);
};
});
}
// Takes a list of usernames and returns a list of user data, with original order preserved.
async function getDetailedCurrentList() {
const keys = currentList.keys();
const sortedKeys = [...keys].sort((a, b) => a - b);
const sortedDetailedList = new Map();
const promises = sortedKeys.map(async (key) => {
const username = currentList.get(key);
const res = await queryDatabaseByUsername(username);
sortedDetailedList.set(key, res);
});
await Promise.all(promises);
return sortedDetailedList;
}
// Get a user's record from database by his username.
async function dumpDatabase() {
if (!db) {
error('The database is not initialized.');
return;
}
const transaction = db.transaction('users', 'readonly');
const objectStore = transaction.objectStore('users');
const request = objectStore.openCursor();
const records = new Map();
return new Promise((resolve) => {
request.onsuccess = (event) => {
const cursor = event.target.result;
if (cursor) {
records.set(cursor.value.rest_id, cursor.value);
cursor.continue();
} else {
// No more results.
resolve(Object.fromEntries(records.entries()));
}
};
request.onerror = (event) => {
error(`Failed to query user ${username} from database.`, event);
resolve(null);
};
});
}
/*
|--------------------------------------------------------------------------
| Twitter API Hooks
|--------------------------------------------------------------------------
*/
// Here we hooks the browser's XHR method to intercept Twitter's Web API calls.
// This need to be done before any XHR request is made.
function hookIntoXHR() {
const originalOpen = unsafeWindow.XMLHttpRequest.prototype.open;
unsafeWindow.XMLHttpRequest.prototype.open = function () {
const url = arguments[1];
// NOTE: This may vary as Twitter upgrades.
// https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers
if (/api\/graphql\/.+\/Followers/.test(url)) {
this.addEventListener('load', function () {
parseTwitterAPIResponse(
this.responseText,
(json) => json.data.user.result.timeline.timeline.instructions
);
});
}
// https://twitter.com/i/api/graphql/kXi37EbqWokFUNypPHhQDQ/BlueVerifiedFollowers
if (/api\/graphql\/.+\/BlueVerifiedFollowers/.test(url)) {
this.addEventListener('load', function () {
parseTwitterAPIResponse(
this.responseText,
(json) => json.data.user.result.timeline.timeline.instructions
);
});
}
// https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following
if (/api\/graphql\/.+\/Following/.test(url)) {
this.addEventListener('load', function () {
parseTwitterAPIResponse(
this.responseText,
(json) => json.data.user.result.timeline.timeline.instructions
);
});
}
// https://twitter.com/i/api/graphql/-5VwQkb7axZIxFkFS44iWw/ListMembers
if (/api\/graphql\/.+\/ListMembers/.test(url)) {
this.addEventListener('load', function () {
parseTwitterAPIResponse(
this.responseText,
(json) => json.data.list.members_timeline.timeline.instructions
);
});
}
// https://twitter.com/i/api/graphql/B9F2680qyuI6keStbcgv6w/ListSubscribers
if (/api\/graphql\/.+\/ListSubscribers/.test(url)) {
this.addEventListener('load', function () {
parseTwitterAPIResponse(
this.responseText,
(json) => json.data.list.subscribers_timeline.timeline.instructions
);
});
}
originalOpen.apply(this, arguments);
};
info('Hooked into XMLHttpRequest.');
}
// We parse the users' information in the API response and write them to the local database.
// The browser's IndexedDB is used to store the data persistently.
function parseTwitterAPIResponse(text, extractor) {
try {
const json = JSON.parse(text);
// NOTE: This may vary as Twitter upgrades.
const instructions = extractor(json);
const entries = instructions.find((item) => item.type === 'TimelineAddEntries').entries;
const users = entries
.filter((item) => item.content.itemContent)
.map((item) => ({
...item.content.itemContent.user_results.result,
entryId: item.entryId,
sortIndex: item.sortIndex,
}));
insertUserDataIntoDatabase(users);
} catch (err) {
error(
`Failed to parse API response. (Message: ${err.message}) ` +
'This may be a problem caused by Twitter updates. Please file an issue on GitHub: ' +
'https://github.com/prinsss/export-twitter-following-list/issues'
);
}
}
/*
|--------------------------------------------------------------------------
| Utility Functions
|--------------------------------------------------------------------------
*/
// Escape characters for CSV file.
function csvEscapeStr(s) {
return `"${s.replace(/\"/g, '""').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`;
}
// Save a text file to disk.
function saveFile(filename, content) {
const link = document.createElement('a');
link.style = 'display: none';
document.body.appendChild(link);
const blob = new Blob([content], { type: 'text/plain' });
const url = URL.createObjectURL(blob);
link.href = url;
link.download = filename;
link.click();
URL.revokeObjectURL(url);
}
// Replace any https://t.co/ link in the string with its corresponding real URL.
function sanitizeProfileDescription(description, urls) {
let str = description;
if (urls?.length) {
for (const { url, expanded_url } of urls) {
str = str.replace(url, expanded_url);
}
}
return str;
}
// Show info logs on both screen and console.
function info(line, ...args) {
console.info('[Export Twitter Following List]', line, ...args);
infoLogs.push(line);
const dom = panelDom ? panelDom.querySelector('#export-logs') : null;
if (dom) {
dom.innerHTML = infoLogs.map((content) => '> ' + String(content)).join('\n');
}
}
// Show error logs on both screen and console.
function error(line, ...args) {
console.error('[Export Twitter Following List]', line, ...args);
errorLogs.push(line);
const dom = panelDom ? panelDom.querySelector('#export-errors') : null;
if (dom) {
dom.innerHTML = errorLogs.map((content) => '> ' + String(content)).join('\n');
}
}
// Show debug logs on console.
function debug(...args) {
console.debug('[Export Twitter Following List]', ...args);
}
})();