/** * The panes as the app sees them: an InterfaceCache is a textarea with a * parse, a dirty bit and a status line, and each kind of pane -- schema, * data (a TurtleCache, which is also where the data source's documents * live), manifest, plugin, JSON -- is a cache class over that. The shape * map's is in ShExShapeMapCache.js, being a page of its own. * * This is doc/ShExCaches.js's source (tsconfig.app.json compiles src/app/ into * doc/); edit here and run `npm run build`. */ class InterfaceCache { // caches for textarea parsers constructor(selection, onLoad) { this._dirty = true; this.selection = selection; this.onLoad = onLoad; this.parsed = null; // a Promise this.url = undefined; // only set if inputarea caches some web resource. this.meta = { prefixes: {}, base: DefaultBase }; } dirty(newVal) { const ret = this._dirty; this._dirty = newVal; return ret; } get() { return this.selection.val(); } // (a manifest or a plugin is also told where it came from and what it is) async set(text, base, _source, _mediaType) { this._dirty = true; this.selection.val(text); // a base this pane was given (?data-base=) outlasts the documents that // come and go in it; a document's own URL is the base otherwise this.meta.base = this.baseOverride || base; if (base !== this.base) { this.url = base; // @@crappyHack1 -- parms should differntiate: // working base: base for URL resolution. // loaded base: place where you can GET current doc. // Note that this.Caches.manifest.set takes a 3rd parm. } } async refresh() { if (!this._dirty) return this.parsed; this.parsed = await this.parse(this.selection.val(), this.meta.base); await this.parsed; this._dirty = false; return this.parsed; } async asyncGet(url) { url = new URL(url, window.location.href).href; const _cache = this; let resp; try { resp = await fetch(url, { headers: { accept: 'text/shex,text/turtle,*/*;q=0.9, test/html;q=0.8', // cache: 'no-cache' -- breaks CORS, so user has to open in new page and force reload there } }); } catch (e) { throw Error("unable to fetch <" + url + ">: " + '\n' + e.message); } if (!resp.ok) throw Error("fetch <" + url + "> got error response " + resp.status + ": " + resp.statusText); const data = await resp.text(); _cache.meta.base = url; try { await _cache.set(data, url, undefined, resp.headers.get('content-type')); } catch (e) { throw Error("error setting " + this.queryStringParm + " with <" + url + ">: " + '\n' + e.message); } $("#loadForm").dialog("close"); return { url: url, data: data }; } callOnLoad() { if (this.onLoad) this.onLoad(); } } class SchemaCache extends InterfaceCache { constructor(selection, onLoad, shexcParser, turtleParser) { super(selection, onLoad); this.shexcParser = shexcParser; this.turtleParser = turtleParser; this.graph = null; this.language = null; this.meta.termToLex = (trm) => trm === ShExWebApp.Validator.Start ? START_SHAPE_LABEL : ShExWebApp.ShExTerm.shExJsTerm2Turtle(trm, this.meta, true); this.meta.lexToTerm = (lex) => lex === START_SHAPE_LABEL ? ShExWebApp.Validator.Start : turtleParser.termToLd(lex, new IRIResolver(this.meta)); } async parse(text, base) { const parseShExR = () => { const graphParser = new ShExWebApp.Validator(this.shexcParser.parseString(ShExWebApp.Util.ShExRSchema, {}, base), // !! do something useful with the meta parm (prefixes and base) ShExWebApp.RdfJsDb(this.graph), {}); // the schema node: the one the schema's URL names (a document that is // also a manifest and data names its schema with a fragment), else the first const schemas = this.graph.getQuads(null, ShExWebApp.Util.RDF.type, "http://www.w3.org/ns/shex#Schema"); const named = schemas.find((q) => q.subject.termType === "NamedNode" && q.subject.value === base); if (!named && schemas.length === 0) throw Error("no shex:Schema in " + base); const schemaRoot = (named || schemas[0]).subject; const val = graphParser.validateNodeShapePair(schemaRoot, ShExWebApp.Validator.Start); // start shape return ShExWebApp.Util.ShExJtoAS(ShExWebApp.Util.ShExRtoShExJ(ShExWebApp.Util.valuesToSchema(ShExWebApp.Util.valToValues(val)))); }; const isJSON = text.match(/^\s*\{/); const isDCTAP = text.match(/\s*shapeID/); this.graph = isJSON ? null : this.tryN3(text, base); this.language = isJSON ? "ShExJ" : isDCTAP ? "DCTAP" : this.graph ? "ShExR" : "ShExC"; $("#results > .status").text("parsing " + this.language + " schema...").show(); const schema = isJSON ? ShExWebApp.Util.ShExJtoAS(JSON.parse(text)) : isDCTAP ? await parseDcTap(text) : this.graph ? parseShExR() : this.shexcParser.parseString(text, this.meta, base); $("#results > .status").hide(); this.callOnLoad(); return schema; async function parseDcTap(text) { const dctap = new ShExWebApp.DcTap(); return await new Promise((resolve, reject) => { $.csv.toArrays(text, {}, (err, data) => { if (err) reject(err); dctap.parseRows(data, base); resolve(dctap.toShEx()); }); }); } } async getItems() { const obj = await this.refresh(); const start = "start" in obj ? [START_SHAPE_LABEL] : []; const rest = "shapes" in obj ? obj.shapes.map((se) => this.meta.termToLex(se.id)) : []; return start.concat(rest); } /** the text as a graph, or null when it is not Turtle: relative IRIs in * a schema written as RDF resolve against the document's base, as ShExC's * do (`this.base` was passed here, and is never set) */ tryN3(text, base) { try { if (text.match(/^\s*$/)) return null; const db = this.turtleParser.parseString(text, this.meta, base); // interpret empty schema as ShExC if (db.getQuads().length === 0) return null; return db; } catch (e) { return null; // signal caller that text isn't Turtle } } } class TurtleCache extends InterfaceCache { constructor(selection, onLoad, turtleParser, queryTrackerController) { super(selection, onLoad); this.turtleParser = turtleParser; this.queryTrackerController = queryTrackerController; this.meta.termToLex = (trm) => ShExWebApp.ShExTerm.rdfJsTerm2Turtle(trm, this.meta); this.meta.lexToTerm = (lex) => turtleParser.termToLd(lex, new IRIResolver(this.meta)); } /** Which neighborhood serves this pane is the modules' business, not this * app's: each says whether it answers to the pane's text and with what * parameters (claimPaneText), and the app builds whichever claims it * (fromParams). A module that declares a `data` parameter -- rdfjs -- * gets the parsed store, since the Turtle parser and this pane's * prefixes and base live here. */ async parse(text, base) { const module = this.neighborhoods.module; const params = this.neighborhoods.params(); // The dirty bit says "something the user touched changed", which is // true of every keystroke in a settings field; whether *this source's* // inputs changed is a different question, and for a source that fetches // its answers rebuilding when they haven't costs a round trip and a // translation for nothing. A local store is rebuilt regardless: it is // cheap, and parsing the document is also how this pane learns its // prefixes and base. const fetches = (module.capabilities || []).length > 0; const signature = JSON.stringify([ShExWebApp.NeighborhoodApi.moduleId(module), params, base, // the tracker is an input too: turning slurp on has to // build a db that reports what it fetches !!this.queryTrackerController.queryTracker]); // What the parameters say, whether or not the db has to be rebuilt from // them: this is bookkeeping about the source, and the shape map's SPARQL // extension reads it. Below the early return it was whatever the last // db that *was* built had left -- so a second entry against the same // endpoint asked its query map with no endpoint recorded. if ("endpoint" in params) this.endpoint = params.endpoint; else delete this.endpoint; if (fetches && this.parsed && signature === this.dbSignature) return this.parsed; this.dbSignature = signature; // A pane of Turtle is this app's to parse: it owns the parser, and the // prefixes and base it finds are what the rest of the app lexifies // nodes with. Panes of anything else go to the module as text. const turtlePane = ShExWebApp.NeighborhoodApi.paneParams(module.dbParams || []) .find((p) => ((p.schema.items || {}).contentMediaType || "") === "text/turtle"); if (turtlePane) params.store = this.turtleParser.parseDocuments(params[turtlePane.name] || [], this.meta, base); const res = module.fromParams(params, this.queryTrackerController.queryTracker); // A db that can go to the network offers a second face which asks with // fetch() rather than with a synchronous XMLHttpRequest. Take it: a // blocking request freezes the tab -- every editor, every button -- for // as long as the endpoint takes to answer, and a wikidata walk makes one // per entity it reaches. Validation awaits it (see invoke). this.callOnLoad(); const db = module.asAsyncDb && typeof res.getNeighborhoodAsync === "function" ? module.asAsyncDb(res) : res; // Neighborhoods return arcs in native order; the "sort quads" control // (#controls, default on) orders them so results don't reshuffle between // runs -- on too when the control is absent, e.g. the worker page. The // order is cosmetic and never affects conformance. const sortQuads = $("#sortQuads").length === 0 || $("#sortQuads").is(":checked"); return sortQuads ? ShExWebApp.NeighborhoodApi.ordered(db) : db; } /** Resolve a query map extension -- SPARQL "SELECT ...", QENTITIES "42" * -- by asking the selected data source, which is the only thing that can * know what the question means. A source that does not offer the * extension says so by name, rather than failing obscurely or running the * question against something that was never configured. */ async resolveQueryMapExtension(language, lexical) { const { queryMapResolverFor, extensionName, moduleId } = ShExWebApp.NeighborhoodApi; const module = this.neighborhoods.module; const resolver = queryMapResolverFor(module, language); if (!resolver) { const e = Error("the QueryMap extension " + extensionName(language) + " is not supported by the neighborhood " + moduleId(module)); e.unsupportedExtension = true; // resolveFixedMap keeps a standing map on this throw e; } // await: a resolver over an endpoint answers with a promise return await resolver.resolve(lexical, await this.refresh()); } /** how a query map extension is written back out: by the name the source * knows it as */ writeQueryMapExtension(language, lexical) { const { queryMapResolverFor, extensionName } = ShExWebApp.NeighborhoodApi; const resolver = queryMapResolverFor(this.neighborhoods.module, language); return (resolver ? resolver.name : extensionName(language)) + " '''" + lexical.replace(/'''/g, "''\\'") + "'''"; } /** candidate focus nodes for the shape-map menus. * * A db that offers its own typeahead (NeighborhoodWebAppDb's optional * suggestFocusNodes) is asked first: it knows what its nodes are, where * this app can only guess by looking at whatever triples are loaded -- * which for the wikidata neighborhood would offer statement and value * nodes alongside the entities anyone would actually validate. */ async getItems() { const data = await this.refresh(); if (typeof data.suggestFocusNodes === "function") return data.suggestFocusNodes("", SPARQL_get_items_limit) .map((suggestion) => this.meta.termToLex(RdfJs.DataFactory.namedNode(suggestion.label))); if (this.endpoint) { const q = "SELECT DISTINCT ?s { ?s ?p ?o } LIMIT " + SPARQL_get_items_limit; // (this read ShEx.Util, which is not a thing in this file: the menu // has been quietly falling back to "no choices found" over endpoints) // ...Promise: a blocking request here freezes the tab while someone is // typing into the menu it fills, which is the worst possible moment const rows = await ShExWebApp.Util.executeQueryPromise(q, this.endpoint, RdfJs.DataFactory); return [MENU_ITEM_materialize].concat(rows.map((row) => this.lexifyFirstColumn(row))); } return data.getQuads().map((t) => this.meta.termToLex(t.subject)); } lexifyFirstColumn(row) { return this.meta.termToLex(row[0]); // row[0] is the first column. } } class ManifestCache extends InterfaceCache { static { this.jsonldP = null; } // manifest-descriptor keys pickSchema/pickData/queryMapLoaded handle // themselves; loadExtraInputs loads the rest static { this.pickLoadedKeys = ["schema", "data", "queryMap"]; } constructor(selection, caches, resultsWidget) { super(selection, null); /** the manifest as it arrived: its text, where from, and which of YAML, * JSON or Turtle it is. The entries the app works from are a reading of * this, never a replacement for it: anything that writes the loaded * manifest back out writes this text, in its own media type */ this.source = null; this.onPickInputs = null; // the app's, to end a debug session standing over inputs about to change this.caches = caches; this.resultsWidget = resultsWidget; this.queryParams = null; // the app's QueryParams registry, assigned post-construction } async set(textOrObj, url, source) { $("#inputSchema .manifest li").remove(); $("#inputData .passes li, #inputData .fails li").remove(); if (typeof textOrObj !== "object") { if (url !== this.base) { this.url = url; // @@crappyHack1 -- parms should differntiate: } try { // exceptions pass through to caller (asyncGet) textOrObj = await this.readManifestText(textOrObj, url); } catch (e) { $("#inputSchema .manifest").append($("
  • ").text(NO_MANIFEST_LOADED)); const throwMe = Error(e + '\n' + textOrObj); throwMe.action = 'load manifest'; throw throwMe; // @@DELME(2017-12-29) // transform deprecated examples.js structure // textOrObj = eval(textOrObj).reduce((acc, schema) => { // function x (data, status) { // return { // schemaLabel: schema.name, // schema: schema.schema, // dataLabel: data.name, // data: data.data, // queryMap: data.queryMap, // status: status // }; // } // return acc.concat( // schema.passes.map(data => x(data, "conformant")), // schema.fails.map(data => x(data, "nonconformant")) // ); // }, []); } } if (!Array.isArray(textOrObj)) textOrObj = [textOrObj]; const demos = textOrObj.reduce((acc, elt) => { if ("action" in elt) { // TODO: move to ShExUtil // compatibility with test suite structure. const action = elt.action; let schemaLabel = action.schema.substr(action.schema.lastIndexOf('/') + 1); let dataLabel = elt["@id"]; let match = null; const emptyGraph = "-- empty graph --"; if ("comment" in elt) { if ((match = elt.comment.match(/^(.*?) \/ { (.*?) }$/))) { schemaLabel = match[1]; dataLabel = match[2] || emptyGraph; } else if ((match = elt.comment.match(/^(.*?) on { (.*?) }$/))) { schemaLabel = match[1]; dataLabel = match[2] || emptyGraph; } else if ((match = elt.comment.match(/^(.*?) as { (.*?) }$/))) { schemaLabel = match[2]; dataLabel = match[1] || emptyGraph; } } const queryMap = "map" in action ? null : ldToTurtle(action.focus, this.caches.inputData.meta.termToLex) + "@" + ("shape" in action ? this.caches.inputSchema.meta.termToLex(action.shape, false) : START_SHAPE_LABEL); const queryMapURL = "map" in action ? action.map : null; elt = Object.assign({ '@id': new URL(elt['@id'], url).href, schemaLabel: schemaLabel, schemaURL: action.schema || url, // dataLabel: "comment" in elt ? elt.comment : (queryMap || dataURL), dataLabel: dataLabel, dataURL: action.data || url }, (queryMap ? { queryMap: queryMap } : { queryMapURL: queryMapURL }), { status: elt["@type"] === "sht:ValidationFailure" ? "nonconformant" : "conformant" }); if ("termResolver" in action || "termResolverURL" in action) { elt.meta = action.termResolver; elt.metaURL = action.termResolverURL || url; } } // `sitematrix` is here because it is a document reference like the // others, and a manifest's references are relative to the *manifest*. // Left out, it was resolved against whichever page loaded the manifest // -- so "../examples/wikidata-sitematrix.json" found the file from // shex-webapp/doc/ and 404'd from extension-map/doc/, which is the same // manifest read by a different app. ["schemaURL", "dataURL", "queryMapURL", "sitematrix"].forEach(parm => { if (parm in elt) { // an entry may name several documents under one key; each is a // reference of its own, not one comma-joined reference elt[parm] = Array.isArray(elt[parm]) ? elt[parm].map((each) => new URL(each, url).href) : new URL(elt[parm], url).href; } else { delete elt[parm]; } }); return acc.concat(elt); }, []); await this.prepareManifest(demos, url); $("#manifestDrop").show(); // may have been hidden if no manifest loaded. } /** the manifest's text as its entries, through @shexjs/manifest: YAML or * JSON -- the classic list, or the YAML-LD form whose @context binds each * vocabulary's `:parms` scope, flattened back to the classic * entries with the plugins collected under `plugins` -- or Turtle, the * same entries read back by predicate, a vocabulary's attributes spelled * through the context the graph names for it. An entry's `schema` * (data, queryMap, overlay) that resolves to a resource becomes * `schemaURL`; one that does not is the document's text. */ async readManifestText(text, url) { const Manifest = ShExWebApp.Manifest; const format = Manifest.detectFormat(text, url); this.source = { text, url, format }; const loadContext = (u) => this.fetchOK(u).then((t) => JSON.parse(t)); if (format === "turtle") { const store = new N3js.Store(); store.addQuads(new N3js.Parser({ baseIRI: url }).parse(text)); // a known vocabulary's context comes from the package's roll-up; an // unknown one is fetched return Manifest.entriesFromGraph(store, { base: url, loadContext }); } let doc; try { doc = JSON.parse(text); } catch (eJson) { try { doc = ShExWebApp.JsYaml.load(text); } catch (eYaml) { throw url.endsWith(".yaml") ? eYaml : eJson; } } // the test suite's manifests are not examples manifests: their entries // (an `action` apiece) are read by set(), as they always were if (doc && !Array.isArray(doc) && Array.isArray(doc["@graph"]) && doc["@graph"][0] && Array.isArray(doc["@graph"][0].entries)) return doc["@graph"][0].entries; const isTestEntry = (e) => e && typeof e === "object" && "action" in e; if (Array.isArray(doc) ? doc.some(isTestEntry) : isTestEntry(doc)) return doc; // an examples manifest: brought up to the canonical form, then read // plainly when that is its JSON-LD reading and as JSON-LD when it is not // (the processor is fetched only then) return Manifest.readJson(doc, { base: url, probe: (u) => this.fetchOK(u).then(() => true, () => false), jsonld: () => ManifestCache.loadJsonLd(), loadContext, onCheck: (check) => { if (check.unknown.length) console.warn(`manifest ${url}: no context defines ${check.unknown.join(", ")}`); }, }); } /** jsonld.js, fetched from beside the app's bundle the first time a * manifest has to be read as JSON-LD */ static loadJsonLd() { if (ManifestCache.jsonldP === null) ManifestCache.jsonldP = new Promise((resolve, reject) => { const w = window; if (w.jsonld) return resolve(w.jsonld); const bundle = document.querySelector('script[src*="webpacks/shex-webapp"]'); const script = document.createElement("script"); script.src = new URL("jsonld.min.js", bundle ? bundle.src : new URL("webpacks/", location.href).href).href; script.onload = () => w.jsonld ? resolve(w.jsonld) : reject(Error(script.src + " did not define jsonld")); script.onerror = () => reject(Error("could not load " + script.src)); document.head.appendChild(script); }); return ManifestCache.jsonldP; } async parse(text, base) { throw Error("should not try to parse manifest cache"); } async getItems() { throw Error("should not try to get manifest cache items"); } maybeGET(obj, base, key, accept) { if (obj[key] != null) { // Take the passed data, guess base if not provided. if (!(key + "URL" in obj)) obj[key + "URL"] = base; obj[key] = Promise.resolve(obj[key]); } else if (key + "URL" in obj) { // absolutize the URL obj[key + "URL"] = this.meta.lexToTerm("<" + obj[key + "URL"] + ">"); // Load the remote resource. obj[key] = new Promise((resolve, reject) => { $.ajax({ accepts: { mycustomtype: accept }, url: this.meta.lexToTerm("<" + obj[key + "URL"] + ">"), dataType: "text" }).then((text) => { resolve(text); }).fail((e) => { this.resultsWidget.append($("
    ").text("Error " + e.status + " " + e.statusText + " on GET " + obj[key + "URL"]).addClass("error"));
                        reject(e);
                    });
                });
            }
            else {
                // Ignore this parameter.
                obj[key] = Promise.resolve(obj[key]);
            }
        }
        async prepareManifest(demoList, base) {
            const listItems = Object.keys(this.caches).reduce((acc, k) => {
                acc[k] = {};
                return acc;
            }, {});
            const nesting = demoList.reduce((acc, elt, idx) => {
                // an entry with no label of its own goes by its name, the shared
                // vocabulary's (@shexjs/manifest reads a classic manifest's title as one)
                const defaultLabel = "name" in elt
                    ? elt.name
                    : `manifest[${idx}]`;
                const schemaLabel = elt.schemaLabel || defaultLabel;
                const key = schemaLabel + "|" + elt.schema;
                if (!(key in acc)) {
                    // first entry with this schema
                    acc[key] = {
                        label: schemaLabel,
                        text: elt.schema,
                        url: elt.schemaURL || (elt.schema ? base : undefined)
                    };
                }
                else {
                    // nth entry with this schema
                }
                if ("dataLabel" in elt || "data" in elt || "dataURL" in elt) {
                    const dataLabel = elt.dataLabel || defaultLabel;
                    const dataEntry = this.makeDataEntry(dataLabel, idx, elt, base);
                    const target = elt.status === "nonconformant"
                        ? "fails"
                        : elt.status === "conformant" ? "passes" : "indeterminant";
                    if (!(target in acc[key])) {
                        // first entry with this data
                        acc[key][target] = [dataEntry];
                    }
                    else {
                        // n'th entry with this data
                        acc[key][target].push(dataEntry);
                    }
                }
                else {
                    // this is a schema-only example
                }
                return acc;
            }, {});
            const nestingAsList = Object.keys(nesting).map(e => nesting[e]);
            await this.paintManifest("#inputSchema .manifest ul", nestingAsList, this.pickSchema.bind(this), listItems, "inputSchema");
        }
        // controls for manifest buttons
        async paintManifest(selector, list, func, listItems, side) {
            $(selector).empty();
            await Promise.all(list.map(async (entry) => {
                // build button disabled and with leading "..." to indicate that it's being loaded
                const button = $("