import { withPage } from "../browser/session.js"; import { cacheKey, readCache, writeCache } from "../cache.js"; import { AccessError, log } from "../errors.js"; import { detectAccessState, extractCompanyPosts, extractEntity, type PublicEntity, } from "../parse/linkedin.js"; import type { Item } from "../types.js"; import { cleanSlug, clip, engagement, isoOrUndefined } from "./shared.js"; /** * What LinkedIn shows a signed-out visitor, and only that. * * Be honest about the ceiling here: a public company or profile page renders, * usually with JSON-LD on it, and the feed does not. LinkedIn also puts up its * auth wall after a handful of pages from one client, with a 200 status, so a * walled read looks exactly like a successful one unless the body is checked. * Every function here reports the access state rather than returning a bare * empty list. */ export interface Entity extends PublicEntity { slug: string; access: "ok" | "walled"; } export async function publicPage(rawSlug: string, kind: "company" | "in"): Promise { const slug = cleanSlug(rawSlug, kind); if (!slug) throw new AccessError("not-found", `"${rawSlug}" is not a LinkedIn ${kind} slug`); const key = cacheKey(["li:entity", kind, slug]); const cached = readCache(key); if (cached) return cached; const url = `https://www.linkedin.com/${kind}/${encodeURIComponent(slug)}/`; const entity = await withPage(url, async (page) => { await settle(page, url); await dismissCookieBanner(page); const html = await page.content(); const state = detectAccessState(html); if (state === "not-found") { throw new AccessError("not-found", `linkedin.com/${kind}/${slug} does not exist.`); } const parsed = extractEntity(html); // A walled page has an `og:title` of its own ("Sign Up | LinkedIn"), so a // name is not evidence that anything was read. The JSON-LD entity is: when // the wall keeps it, the summary is real and thin; when it does not, there // is nothing here but the wall wearing the page's clothes. if (state === "walled" && parsed.kind === "unknown") { throw new AccessError( "walled", `LinkedIn put up its sign-in wall on /${kind}/${slug}. This server never signs in. Wait ` + "several minutes before reading LinkedIn again, and read fewer pages per run.", ); } // A walled page often still carries the JSON-LD summary. Return it, and say // it was walled, so nobody mistakes the thin version for the whole page. return { ...parsed, slug, access: state === "walled" ? "walled" : "ok" } satisfies Entity; }); writeCache(key, entity); return entity; } export async function companyPosts(rawSlug: string, limit: number): Promise { const slug = cleanSlug(rawSlug, "company"); const key = cacheKey(["li:posts", slug, limit]); const cached = readCache(key); if (cached) return cached; const url = `https://www.linkedin.com/company/${encodeURIComponent(slug)}/posts/`; const items = await withPage(url, async (page) => { await settle(page, url); await dismissCookieBanner(page); const state = detectAccessState(await page.content()); if (state === "walled") { throw new AccessError( "walled", `LinkedIn serves /company/${slug}/posts/ only to signed-in visitors right now. The company ` + "page itself may still be readable - try linkedin_company.", ); } const scraped = await page.evaluate(extractCompanyPosts); return scraped.slice(0, limit).map((post, index) => ({ source: "linkedin", externalId: post.urn ?? post.url ?? `${slug}:${index}`, url: post.url ?? url, author: slug, text: clip(post.text), publishedAt: isoOrUndefined(post.publishedAt), engagement: engagement({ reactions: post.reactions, comments: post.comments }), via: `linkedin.com/company/${slug}`, })); }); log("linkedin", `${slug}: ${items.length} posts`); writeCache(key, items); return items; } /** * LinkedIn answers the first request with a redirect - to a locale host, or to * its wall - so `domcontentloaded` resolves on a document that is already being * thrown away. Wait for the one that survives, and read it as HTML rather than * evaluating inside it, which is what makes this immune to the redirect rather * than merely lucky about it. * * Waiting is not retrying: nothing here asks again for a page LinkedIn declined. */ async function settle(page: import("playwright").Page, url: string) { await page.goto(url, { waitUntil: "load", timeout: 30_000 }); await page.waitForLoadState("domcontentloaded").catch(() => undefined); } /** * Decline the non-essential half of the consent banner. Left alone it covers * the page and the parser reads a dialog instead of a profile. */ async function dismissCookieBanner(page: import("playwright").Page) { const reject = page .locator('button:has-text("Reject"), button[action-type="DENY"], [data-test-id="cookie-reject"]') .first(); if (await reject.isVisible({ timeout: 1_500 }).catch(() => false)) { await reject.click({ timeout: 2_000 }).catch(() => undefined); } }