using System.Collections.Concurrent; using System.Text; using System.Text.Json.Nodes; namespace EcencyApi.Infrastructure; /// /// JSON-RPC client for Hive nodes with health-aware failover. The per-node /// health state and pool ordering live in /// (shared with the Hive-Engine client); this class adds the Hive call /// semantics: /// /// - RPC-level errors (JSON error field) surface immediately without failover /// (dhive semantics — an application error, not an unhealthy node). /// - Overload statuses (429/502/503/504) advance to the next node immediately /// instead of burning a same-node retry. /// - An `unknown method` answer to a call that names a fallback method asks /// the same node the fallback, and marks the node for a while (CallMethod). /// /// Latency is tracked per call class (see ) because the /// reads made through this client are bimodal in cost. Failure state is not: it /// stays node-wide. /// /// Not adopted (overkill at proxy call rates): request hedging, per-API failure /// profiles, and head-block staleness checks. /// public sealed class HiveRpcClient { private readonly string[] _nodes; private readonly int _timeoutMs; private readonly int _failoverThreshold; private readonly NodeHealthTracker _health; private readonly Func _clock; private int _seq; // Per node, the methods it answered `unknown method` for and until when (on // _clock). Bounded by nodes x fallback-capable methods; see CallMethod. private readonly ConcurrentDictionary<(int Node, string Method), long> _unsupportedUntil = new(); // timeoutMs 2000 / failoverThreshold 2 mirror the dhive Client options the // Node service constructed its clients with. public HiveRpcClient(string[] nodes, int timeoutMs = 2000, int failoverThreshold = 2) : this(nodes, timeoutMs, failoverThreshold, null) { } /// Test seam for the health tracker's notion of time. internal HiveRpcClient(string[] nodes, int timeoutMs, int failoverThreshold, Func? clock) { _nodes = nodes; _timeoutMs = timeoutMs; _failoverThreshold = Math.Max(1, failoverThreshold); _health = new NodeHealthTracker(nodes.Length, clock); _clock = clock ?? (() => Environment.TickCount64); } public IReadOnlyList Nodes => _nodes; /// /// Per-node health for an internal stats endpoint: which node carries the /// traffic, which one times out, which one is parked. Hosts only; these are /// the public node names, nothing about this deployment. /// public JsonArray HealthSnapshot() { var arr = new JsonArray(); foreach (var v in _health.Snapshot()) { var node = new JsonObject { ["node"] = Uri.TryCreate(_nodes[v.Index], UriKind.Absolute, out var u) ? u.Host : _nodes[v.Index], ["calls"] = v.Calls, ["ok"] = v.Successes, ["failures"] = v.Failures, ["timeouts"] = v.Timeouts, ["rate_limited"] = v.RateLimits, }; // One EWMA per call class (see CallClass): the pool is ordered from // the profile of the class being called, so both must be readable to // tell "this node is slow" from "this node is slow at feed queries". // ewma_ms/samples stay the cheap class, which is what they have always // reported in practice, since cheap calls dominate by count. var cheap = v.Latency.First(l => l.Class == CallClass.Cheap); var heavy = v.Latency.First(l => l.Class == CallClass.Heavy); node["ewma_ms"] = cheap.EwmaMs is { } ce ? JsonValue.Create(Math.Round(ce, 1)) : null; node["samples"] = cheap.Samples; node["heavy_ewma_ms"] = heavy.EwmaMs is { } he ? JsonValue.Create(Math.Round(he, 1)) : null; node["heavy_samples"] = heavy.Samples; node["consecutive_failures"] = v.ConsecutiveFailures; node["recent_failure"] = v.RecentFailure; node["rate_limited_for_ms"] = v.RateLimitedForMs; node["parked_for_ms"] = v.FailureParkedForMs; node["failure_rate"] = Math.Round(v.FailureRate, 3); // Methods this node is asked the fallback for until its mark lapses. var unsupported = new JsonArray(); foreach (var m in UnsupportedMethods(v.Index)) unsupported.Add(m); node["unsupported"] = unsupported; arr.Add(node); } return arr; } public sealed class RpcException : Exception { public RpcException(string message) : base(message) { } } private static long NowMs => Environment.TickCount64; /// How many nodes may be consulted to satisfy a soft result /// preference before the first well-formed answer is accepted as-is. private const int MaxPreferenceProbes = 2; /// How long a node that answered `unknown method` is asked the /// fallback instead. Long enough that a node lacking the method costs one /// wasted round trip per method per half hour, short enough that a node /// upgraded in place is used for the new method again within the half hour, /// with no restart. internal const long UnsupportedMethodTtlMs = 30 * 60 * 1000; // ---- calls ------------------------------------------------------------- /// Optional shape check for the RPC result. A node /// can return 200 with valid JSON that carries neither an error nor a usable /// result (observed in production as multi-hour windows of get_accounts /// yielding no array); without validation that response counts as a SUCCESS, /// so the health tracker keeps the poisoned node ranked first for the whole /// window. A failed check is treated as node failure and fails over. /// Optional *soft* check: the result is well-formed /// but this node cannot serve the caller's needs. Unlike validateResult this is /// not a health signal — the node is fine for other calls — so it is neither /// retried nor marked unhealthy; we simply move on and keep its answer. If no /// node satisfies the preference, the first such answer is returned rather than /// throwing, so the caller is never worse off than without the preference. /// /// Exists because some Hive nodes serve accounts with account metadata stripped: /// balances and reputation are correct, posting_json_metadata is empty. That is a /// valid 200 with a usable array, so shape validation passes and the latency EWMA /// keeps such a node ranked first — silently blanking every metadata-derived /// feature (portfolio engine/chain token visibility) with no error and no log. /// Which latency profile this call's timings belong /// to. That profile is also the one the pool is ordered from for this call. /// Defaults to Cheap, so a caller that makes one shape of call keeps exactly /// one profile per node. public Task Call(string api, string method, JsonNode @params, Func? validateResult = null, Func? preferResult = null, CallClass callClass = CallClass.Cheap) { // The legacy `call` envelope the Node service always sent. hived resolves // it for its own APIs (condenser_api, database_api); hivemind's `bridge` // is not one of them, so bridge reads must use CallMethod. var request = new JsonObject { ["id"] = Interlocked.Increment(ref _seq), ["jsonrpc"] = "2.0", ["method"] = "call", ["params"] = new JsonArray(api, method, @params), }; return Send(request, method, validateResult, preferResult, callClass); } /// /// The modern JSON-RPC form, `"method": "bridge.get_post"` with the params /// as given, which jussi/HAF route to hived or hivemind by prefix. Needed /// for every hivemind (`bridge`) read; works for condenser_api too. /// /// An older method serving the same read, for nodes /// that do not have this one yet. A node that answers `unknown method` is not /// failed over: it is asked the fallback at once, in the same attempt, and is /// asked the fallback directly for . /// Whatever that node answers is returned as is. public Task CallMethod(string qualifiedMethod, JsonNode @params, Func? validateResult = null, Func? preferResult = null, CallClass callClass = CallClass.Cheap, (string Method, JsonNode Params)? fallback = null) { var request = Envelope(qualifiedMethod, @params); var fallbackRequest = fallback is { } f ? Envelope(f.Method, f.Params) : null; return Send(request, qualifiedMethod, validateResult, preferResult, callClass, fallbackRequest); } private JsonObject Envelope(string qualifiedMethod, JsonNode @params) => new() { ["id"] = Interlocked.Increment(ref _seq), ["jsonrpc"] = "2.0", ["method"] = qualifiedMethod, ["params"] = @params, }; // hivemind without the method: HTTP 200, code -32602, "Assert // Exception:unknown method: bridge_apiget_post_v2" (verified 2026-11-12). // The same code and prefix carry asserts that echo params ("Post a/x does // not exist", "invalid tag `x`"), so the assert has to START with the // unknown-method text and name the method asked for; otherwise one crafted // permlink would mark a capable node for the whole TTL. private const string UnknownMethodPrefix = "Assert Exception:unknown method:"; internal static bool IsUnknownMethod(string message, string qualifiedMethod) { if (!message.StartsWith(UnknownMethodPrefix, StringComparison.Ordinal)) return false; var named = message[UnknownMethodPrefix.Length..].Trim(); var bare = qualifiedMethod[(qualifiedMethod.LastIndexOf('.') + 1)..]; // hivemind names it `_api`: bridge.get_post_v2 is // bridge_apiget_post_v2. return named.EndsWith("_api" + bare, StringComparison.Ordinal); } private bool Unsupported(int nodeIndex, string method) { if (!_unsupportedUntil.TryGetValue((nodeIndex, method), out var until)) return false; if (_clock() < until) return true; _unsupportedUntil.TryRemove(new KeyValuePair<(int, string), long>((nodeIndex, method), until)); return false; } private IEnumerable UnsupportedMethods(int nodeIndex) => _unsupportedUntil.Keys.Where(k => k.Node == nodeIndex && Unsupported(nodeIndex, k.Method)) .Select(k => k.Method) .OrderBy(m => m, StringComparer.Ordinal) .ToArray(); private async Task Send(JsonObject request, string method, Func? validateResult, Func? preferResult, CallClass callClass, JsonObject? fallbackRequest = null) { // JsJson: a lone-surrogate username from a client token must serialize // (JSON.stringify semantics) instead of throwing in the writer. var body = JsJson.Stringify(request); var fallbackBody = fallbackRequest is null ? null : JsJson.Stringify(fallbackRequest); Exception? lastError = null; JsonNode? unpreferred = null; var haveUnpreferred = false; var unpreferredCount = 0; foreach (var nodeIndex in _health.OrderedNodeIndices(callClass)) { var node = _nodes[nodeIndex]; for (var attempt = 0; attempt < _failoverThreshold; attempt++) { // The ordering above is a snapshot; admission is decided now. if (!_health.TryBeginAttempt(nodeIndex)) break; var started = NowMs; try { var useFallback = fallbackBody != null && Unsupported(nodeIndex, method); JsonNode? result; try { result = await CallNode(node, useFallback ? fallbackBody! : body); } catch (RpcException e) when (fallbackBody != null && !useFallback && IsUnknownMethod(e.Message, method)) { // The node is up, it just predates the method: same node, // older method, and remember it. Not a failure, not a failover. // Two round trips against the caller's budget, but only once // per (node, method) per TTL, and inside a fill the fill gate // already bounds. _unsupportedUntil[(nodeIndex, method)] = _clock() + UnsupportedMethodTtlMs; // Rank the node on the v1 answer it will give from now on, // not on the probe that does not recur within the TTL. started = NowMs; result = await CallNode(node, fallbackBody); } if (validateResult != null && !validateResult(result)) { // Don't burn a same-node retry: a node serving malformed // 200s keeps serving them (observed for hours at a time). throw new NodeUnavailableException( $"RPC node {node} returned unusable {method} result", advanceImmediately: true); } // The node is healthy either way — record the success before // deciding whether its answer is the one we wanted. _health.RecordSuccess(nodeIndex, NowMs - started, callClass); if (preferResult != null && !preferResult(result)) { // Keep the first such answer as the floor and try the next // node; same-node retry would return the same thing. if (!haveUnpreferred) { unpreferred = result; haveUnpreferred = true; } // Bounded on purpose. Roughly an eighth of active accounts // genuinely carry no metadata, and for those NO node can // satisfy the preference — probing the whole pool every time // would multiply RPC load on a common case to route around a // rare one. One alternative is enough to get past a single // metadata-stripping node, which is all this guards against. if (++unpreferredCount >= MaxPreferenceProbes) return unpreferred; break; } return result; } catch (RpcException) { // The node answered; the error is the application's. No // failover (dhive semantics), and no failure mark. _health.RecordSuccess(nodeIndex, NowMs - started, callClass); // ...but if we only came to this node to improve on an answer we // already hold, its error belongs to the optional probe, not to // the caller's request. Rethrowing here would fail a call that // would have succeeded without the preference. if (haveUnpreferred) return unpreferred; throw; } catch (NodeUnavailableException e) { lastError = e; if (e.IsRateLimit) { _health.RecordRateLimited(nodeIndex, e.RetryAfterMs); } else if (_health.RecordFailure(nodeIndex, NowMs - started, callClass, e.IsTimeout)) { break; // this failure parked the node: no same-node retry } if (e.AdvanceImmediately) { break; // don't retry a throttled/overloaded node } } catch (Exception e) { lastError = e; if (_health.RecordFailure(nodeIndex, NowMs - started, callClass)) { break; } } finally { _health.EndAttempt(nodeIndex); } } } // No node satisfied the preference, but one answered well-formed: that is // the normal outcome when the preference is genuinely unsatisfiable (an // account really has no metadata), so return it instead of failing. if (haveUnpreferred) return unpreferred; // Every node exhausted — surface the last transport error (dhive throws // after cycling the whole list). throw lastError ?? new InvalidOperationException("no RPC nodes configured"); } /// A node is unhealthy/unreachable; try the next one. Distinct from /// RpcException (a real RPC-level error that must not fail over). private sealed class NodeUnavailableException : Exception { public bool AdvanceImmediately { get; } public bool IsRateLimit { get; } public bool IsTimeout { get; } public int? RetryAfterMs { get; } public Exception? Cause { get; private set; } public NodeUnavailableException(string message, bool advanceImmediately, bool isRateLimit = false, int? retryAfterMs = null, bool isTimeout = false) : base(message) { AdvanceImmediately = advanceImmediately; IsRateLimit = isRateLimit; RetryAfterMs = retryAfterMs; IsTimeout = isTimeout; } public NodeUnavailableException WithInner(Exception inner) { Cause = inner; return this; } } // Overload statuses mean "this node is throttling/failing at the edge" — // skip it now rather than burning a retry that will fail the same way. private static bool IsOverloadStatus(int status) => status is 429 or 502 or 503 or 504; private async Task CallNode(string node, string body) { using var req = new HttpRequestMessage(HttpMethod.Post, node) { Content = new StringContent(body, Encoding.UTF8, "application/json"), }; using var cts = new CancellationTokenSource(_timeoutMs); HttpResponseMessage resp; try { resp = await Upstream.Http.SendAsync(req, cts.Token); } catch (OperationCanceledException e) when (cts.IsCancellationRequested) { throw new NodeUnavailableException($"RPC node {node} timed out", advanceImmediately: false, isTimeout: true).WithInner(e); } catch (HttpRequestException e) { throw new NodeUnavailableException($"RPC node {node} unreachable: {e.Message}", advanceImmediately: false).WithInner(e); } using (resp) { if (!resp.IsSuccessStatusCode) { var status = (int)resp.StatusCode; var retryAfter = status == 429 ? NodeHealthTracker.ParseRetryAfterMs(resp.Headers.TryGetValues("Retry-After", out var vals) ? vals.FirstOrDefault() : null) : null; throw new NodeUnavailableException( $"RPC node {node} returned {status}", advanceImmediately: IsOverloadStatus(status), isRateLimit: status == 429, retryAfterMs: retryAfter); } var text = await resp.Content.ReadAsStringAsync(cts.Token); JsonNode? parsed; try { parsed = JsonNode.Parse(text); } catch (System.Text.Json.JsonException e) { throw new NodeUnavailableException($"RPC node {node} returned non-JSON", advanceImmediately: false).WithInner(e); } if (parsed is JsonObject obj && obj.TryGetPropertyValue("error", out var error) && error != null) { var message = error["message"]?.GetValue() ?? error.ToJsonString(); throw new RpcException(message); } return parsed?["result"]; } } // ---- typed helpers matching the dhive calls the handlers make -------- public async Task GetAccounts(IEnumerable names) { var nameArr = new JsonArray(); foreach (var n in names) { // A null name serializes to JSON null (matches dhive: getAccounts([undefined]) // -> params [null]). Hive's get_accounts(["null"]) is a real account (@null); // get_accounts([null]) is empty — so this distinction is load-bearing. nameArr.Add(n is null ? null : JsonValue.Create(n)); } var result = await Call("condenser_api", "get_accounts", new JsonArray(nameArr), IsAccountArray, HasAnyAccountMetadata); return result as JsonArray; } /// /// A usable get_accounts result: an array whose entries are account objects or /// JSON null (an unknown account). A node answering with scalar entries passes a /// bare "is an array" check but yields nothing readable downstream — metadata /// reads off it come back empty, which silently blanks portfolio token /// visibility exactly like a metadata-stripping node. Treat it as node failure. /// internal static bool IsAccountArray(JsonNode? result) { if (result is not JsonArray accounts) return false; foreach (var account in accounts) { if (account is not null and not JsonObject) return false; } return true; } /// /// True when at least one returned account carries a non-empty /// posting_json_metadata. Nodes that strip account metadata answer with a /// well-formed array whose entries have it blank; portfolio token visibility /// is derived from that field, so such an answer silently reads as "this user /// enabled nothing". Soft preference, not a health signal: an account that /// genuinely has no metadata produces the same shape, and after every node /// declines the caller still gets the response. /// internal static bool HasAnyAccountMetadata(JsonNode? result) { if (result is not JsonArray accounts || accounts.Count == 0) return true; var sawAccount = false; foreach (var account in accounts) { if (account is not JsonObject) continue; sawAccount = true; var meta = JsVal.AsString(JsVal.Prop(account, "posting_json_metadata")); if (!string.IsNullOrEmpty(meta)) return true; } // An all-null array (unknown account) has nothing to prefer either way. return !sawAccount; } public Task GetDynamicGlobalProperties() => Call("condenser_api", "get_dynamic_global_properties", new JsonArray(), r => r is JsonObject); } /// /// The shared RPC client. The Node service built two dhive Clients /// (private-api.ts and hive-explorer.ts, the latter with hapi.ecency.com /// first) — that node has since been decommissioned, so both collapse into /// one client with a single shared health state. /// public static class HiveClients { // techcoderx.com and hiveapi.actifit.io are deliberately absent: both serve // accounts with posting_json_metadata stripped (balances correct, metadata // empty). They are fast, so the latency EWMA ranked them first and the // portfolio engine/chain layers came back empty for everyone. GetAccounts // also routes around such a node at runtime, but keeping them out of the pool // means correctness here does not depend on that fallback firing. // hive-api.arcange.eu and hive-api.3speak.tv are absent for the same reason // as each other: neither completes a TCP connect from any host this service // runs on (SYN, no answer, on 443 and on 80), so every attempt cost the full // per-node timeout and, in bursts, took every in-flight fill with it. A node // that answers nothing is worse for the pool than a slow one, because the // health tracker learns latency from answers and has none to learn from: it // parks after three consecutive failures, the park lapses, it is probed // again. While unparked it sits in config order, which is where any // unproven latency profile starts. public static readonly IReadOnlyList DefaultNodes = new[] { "https://api.hive.blog", "https://api.deathwing.me", "https://rpc.mahdiyari.info", "https://api.openhive.network", "https://api.syncad.com", "https://api.c0ff33a.uk", }; public static readonly HiveRpcClient Default = new(DefaultNodes.ToArray()); }