648 lines
19 KiB
TypeScript
648 lines
19 KiB
TypeScript
// @ts-nocheck
|
|
import { action, internalMutation, query } from "./_generated/server"
|
|
import { api, internal } from "./_generated/api"
|
|
import { v } from "convex/values"
|
|
|
|
const POLL_KEY = "manual-parts"
|
|
const LISTING_EXPIRES_MS = 14 * 24 * 60 * 60 * 1000
|
|
const STALE_AFTER_MS = 36 * 60 * 60 * 1000
|
|
const BASE_REFRESH_MS = 24 * 60 * 60 * 1000
|
|
const MAX_BACKOFF_MS = 7 * 24 * 60 * 60 * 1000
|
|
const MAX_QUERIES_PER_RUN = 4
|
|
const MAX_RESULTS_PER_QUERY = 8
|
|
const MAX_UNIQUE_RESULTS = 48
|
|
|
|
const SYNTHETIC_ITEM_PREFIX = "123456789"
|
|
const PLACEHOLDER_IMAGE_HOSTS = [
|
|
"images.unsplash.com",
|
|
"via.placeholder.com",
|
|
"placehold.co",
|
|
] as const
|
|
|
|
const POLL_QUERY_POOL = [
|
|
{
|
|
label: "dixie narco part number",
|
|
keywords: "dixie narco vending part number",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "crane national vendors part",
|
|
keywords: "crane national vendors vending part",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "seaga vending control board",
|
|
keywords: "seaga vending control board",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "coinco coin mech",
|
|
keywords: "coinco vending coin mech",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "mei bill validator",
|
|
keywords: "mei vending bill validator",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "wittern delivery motor",
|
|
keywords: "wittern vending delivery motor",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "vending refrigeration deck",
|
|
keywords: "vending machine refrigeration deck",
|
|
categoryId: "11700",
|
|
},
|
|
{
|
|
label: "vending keypad",
|
|
keywords: "vending machine keypad",
|
|
categoryId: "11700",
|
|
},
|
|
] as const
|
|
|
|
function normalizeText(value: string): string {
|
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").replace(/\s+/g, " ").trim()
|
|
}
|
|
|
|
function parsePositiveNumber(value: string): number | null {
|
|
const match = value.match(/([0-9]+(?:\.[0-9]+)?)/)
|
|
if (!match) {
|
|
return null
|
|
}
|
|
|
|
const parsed = Number.parseFloat(match[1])
|
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
return null
|
|
}
|
|
|
|
return parsed
|
|
}
|
|
|
|
function getPollQueriesForRun(now: number) {
|
|
const total = POLL_QUERY_POOL.length
|
|
if (total === 0) {
|
|
return []
|
|
}
|
|
|
|
const startIndex = Math.floor(now / BASE_REFRESH_MS) % total
|
|
const count = Math.min(MAX_QUERIES_PER_RUN, total)
|
|
const queries: (typeof POLL_QUERY_POOL)[number][] = []
|
|
|
|
for (let index = 0; index < count; index += 1) {
|
|
queries.push(POLL_QUERY_POOL[(startIndex + index) % total])
|
|
}
|
|
|
|
return queries
|
|
}
|
|
|
|
function buildAffiliateLink(viewItemUrl: string): string {
|
|
const campaignId = process.env.EBAY_AFFILIATE_CAMPAIGN_ID?.trim() || ""
|
|
if (!campaignId) {
|
|
return viewItemUrl
|
|
}
|
|
|
|
try {
|
|
const url = new URL(viewItemUrl)
|
|
url.searchParams.set("mkcid", "1")
|
|
url.searchParams.set("mkrid", "711-53200-19255-0")
|
|
url.searchParams.set("siteid", "0")
|
|
url.searchParams.set("campid", campaignId)
|
|
url.searchParams.set("toolid", "10001")
|
|
url.searchParams.set("mkevt", "1")
|
|
return url.toString()
|
|
} catch {
|
|
return viewItemUrl
|
|
}
|
|
}
|
|
|
|
function firstValue<T>(value: T | T[] | undefined): T | undefined {
|
|
if (value === undefined) {
|
|
return undefined
|
|
}
|
|
return Array.isArray(value) ? value[0] : value
|
|
}
|
|
|
|
function parseUrl(value: string): URL | null {
|
|
try {
|
|
return new URL(value)
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
function isTrustedListingCandidate(listing: ReturnType<typeof normalizeEbayItem>) {
|
|
const itemId = listing.itemId?.trim() || ""
|
|
if (!/^[0-9]{9,15}$/.test(itemId)) {
|
|
return false
|
|
}
|
|
|
|
if (itemId.startsWith(SYNTHETIC_ITEM_PREFIX)) {
|
|
return false
|
|
}
|
|
|
|
const parsedUrl = parseUrl(listing.viewItemUrl || "")
|
|
if (!parsedUrl) {
|
|
return false
|
|
}
|
|
|
|
const host = parsedUrl.hostname.toLowerCase()
|
|
if (!host.includes("ebay.")) {
|
|
return false
|
|
}
|
|
|
|
if (!parsedUrl.pathname.includes("/itm/")) {
|
|
return false
|
|
}
|
|
|
|
const parsedPrice = parsePositiveNumber(listing.price || "")
|
|
if (!parsedPrice) {
|
|
return false
|
|
}
|
|
|
|
if (listing.imageUrl) {
|
|
const parsedImage = parseUrl(listing.imageUrl)
|
|
const imageHost = parsedImage?.hostname.toLowerCase() || ""
|
|
if (PLACEHOLDER_IMAGE_HOSTS.some((placeholder) => imageHost.includes(placeholder))) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
function isRateLimitError(message: string): boolean {
|
|
const normalized = message.toLowerCase()
|
|
return (
|
|
normalized.includes("10001") ||
|
|
normalized.includes("rate limit") ||
|
|
normalized.includes("too many requests") ||
|
|
normalized.includes("exceeded the number of times") ||
|
|
normalized.includes("quota")
|
|
)
|
|
}
|
|
|
|
function getBackoffMs(consecutiveFailures: number, rateLimited: boolean) {
|
|
if (rateLimited) {
|
|
return BASE_REFRESH_MS
|
|
}
|
|
|
|
const base = BASE_REFRESH_MS / 2
|
|
const multiplier = Math.max(1, consecutiveFailures)
|
|
return Math.min(base * multiplier, MAX_BACKOFF_MS)
|
|
}
|
|
|
|
async function readEbayErrorMessage(response: Response) {
|
|
const text = await response.text().catch(() => "")
|
|
if (!text.trim()) {
|
|
return `eBay API error: ${response.status}`
|
|
}
|
|
|
|
try {
|
|
const parsed = JSON.parse(text) as any
|
|
const messages = parsed?.errorMessage?.[0]?.error?.[0]
|
|
const message = Array.isArray(messages?.message)
|
|
? messages.message[0]
|
|
: messages?.message
|
|
|
|
if (typeof message === "string" && message.trim()) {
|
|
const errorId = Array.isArray(messages?.errorId)
|
|
? messages.errorId[0]
|
|
: messages?.errorId
|
|
return errorId
|
|
? `eBay API error ${errorId}: ${message}`
|
|
: `eBay API error: ${message}`
|
|
}
|
|
} catch {
|
|
// Fall through to returning raw text.
|
|
}
|
|
|
|
return text.trim() || `eBay API error: ${response.status}`
|
|
}
|
|
|
|
function readEbayJsonError(data: any): string | null {
|
|
const errorMessage = data?.errorMessage?.[0]?.error?.[0]
|
|
const message = Array.isArray(errorMessage?.message)
|
|
? errorMessage.message[0]
|
|
: errorMessage?.message
|
|
|
|
if (typeof message !== "string" || !message.trim()) {
|
|
return null
|
|
}
|
|
|
|
const errorId = Array.isArray(errorMessage?.errorId)
|
|
? errorMessage.errorId[0]
|
|
: errorMessage?.errorId
|
|
|
|
return errorId
|
|
? `eBay API error ${errorId}: ${message}`
|
|
: `eBay API error: ${message}`
|
|
}
|
|
|
|
function normalizeEbayItem(item: any, fetchedAt: number) {
|
|
const currentPrice = firstValue(item?.sellingStatus?.currentPrice)
|
|
const shippingCost = firstValue(item?.shippingInfo?.shippingServiceCost)
|
|
const condition = firstValue(item?.condition)
|
|
const viewItemUrl = item?.viewItemURL || item?.viewItemUrl || ""
|
|
const title = item?.title || "Unknown Item"
|
|
|
|
return {
|
|
itemId: String(item?.itemId || ""),
|
|
title,
|
|
normalizedTitle: normalizeText(title),
|
|
price: `${currentPrice?.value || "0"} ${currentPrice?.currencyId || "USD"}`,
|
|
currency: currentPrice?.currencyId || "USD",
|
|
imageUrl: item?.galleryURL || undefined,
|
|
viewItemUrl,
|
|
condition: condition?.conditionDisplayName || undefined,
|
|
shippingCost: shippingCost?.value
|
|
? `${shippingCost.value} ${shippingCost.currencyId || currentPrice?.currencyId || "USD"}`
|
|
: undefined,
|
|
affiliateLink: buildAffiliateLink(viewItemUrl),
|
|
sourceQueries: [] as string[],
|
|
fetchedAt,
|
|
firstSeenAt: fetchedAt,
|
|
lastSeenAt: fetchedAt,
|
|
expiresAt: fetchedAt + LISTING_EXPIRES_MS,
|
|
active: true,
|
|
}
|
|
}
|
|
|
|
async function searchEbayListings(query: (typeof POLL_QUERY_POOL)[number]) {
|
|
const appId = process.env.EBAY_APP_ID?.trim()
|
|
if (!appId) {
|
|
throw new Error("eBay App ID is not configured")
|
|
}
|
|
|
|
const url = new URL("https://svcs.ebay.com/services/search/FindingService/v1")
|
|
url.searchParams.set("OPERATION-NAME", "findItemsAdvanced")
|
|
url.searchParams.set("SERVICE-VERSION", "1.0.0")
|
|
url.searchParams.set("SECURITY-APPNAME", appId)
|
|
url.searchParams.set("RESPONSE-DATA-FORMAT", "JSON")
|
|
url.searchParams.set("REST-PAYLOAD", "true")
|
|
url.searchParams.set("keywords", query.keywords)
|
|
url.searchParams.set("sortOrder", "BestMatch")
|
|
url.searchParams.set("paginationInput.entriesPerPage", String(MAX_RESULTS_PER_QUERY))
|
|
if (query.categoryId) {
|
|
url.searchParams.set("categoryId", query.categoryId)
|
|
}
|
|
|
|
const response = await fetch(url.toString(), {
|
|
method: "GET",
|
|
headers: {
|
|
Accept: "application/json",
|
|
},
|
|
})
|
|
|
|
if (!response.ok) {
|
|
throw new Error(await readEbayErrorMessage(response))
|
|
}
|
|
|
|
const data = await response.json()
|
|
const jsonError = readEbayJsonError(data)
|
|
if (jsonError) {
|
|
throw new Error(jsonError)
|
|
}
|
|
const findItemsAdvancedResponse = data.findItemsAdvancedResponse?.[0]
|
|
const searchResult = findItemsAdvancedResponse?.searchResult?.[0]
|
|
const items = Array.isArray(searchResult?.item)
|
|
? searchResult.item
|
|
: searchResult?.item
|
|
? [searchResult.item]
|
|
: []
|
|
|
|
return items
|
|
}
|
|
|
|
export const getCacheOverview = query({
|
|
args: {},
|
|
handler: async (ctx) => {
|
|
const now = Date.now()
|
|
const state =
|
|
(await ctx.db
|
|
.query("ebayPollState")
|
|
.withIndex("by_key", (q) => q.eq("key", POLL_KEY))
|
|
.unique()) || null
|
|
|
|
const listings = await ctx.db.query("ebayListings").collect()
|
|
const activeListings = listings.filter(
|
|
(listing) => listing.active && listing.expiresAt >= now
|
|
)
|
|
|
|
const freshnessMs = state?.lastSuccessfulAt
|
|
? Math.max(0, now - state.lastSuccessfulAt)
|
|
: null
|
|
|
|
return {
|
|
key: POLL_KEY,
|
|
status: state?.status || "idle",
|
|
lastSuccessfulAt: state?.lastSuccessfulAt || null,
|
|
lastAttemptAt: state?.lastAttemptAt || null,
|
|
nextEligibleAt: state?.nextEligibleAt || null,
|
|
lastError: state?.lastError || null,
|
|
consecutiveFailures: state?.consecutiveFailures || 0,
|
|
queryCount: state?.queryCount || 0,
|
|
itemCount: state?.itemCount || 0,
|
|
sourceQueries: state?.sourceQueries || [],
|
|
freshnessMs,
|
|
isStale: freshnessMs !== null ? freshnessMs >= STALE_AFTER_MS : true,
|
|
listingCount: listings.length,
|
|
activeListingCount: activeListings.length,
|
|
}
|
|
},
|
|
})
|
|
|
|
export const listCachedListings = query({
|
|
args: {
|
|
limit: v.optional(v.number()),
|
|
},
|
|
handler: async (ctx, args) => {
|
|
const now = Date.now()
|
|
const listings = await ctx.db.query("ebayListings").collect()
|
|
const normalized = listings.map((listing) => ({
|
|
...listing,
|
|
active: listing.active && listing.expiresAt >= now,
|
|
}))
|
|
|
|
normalized.sort((a, b) => {
|
|
if (a.active !== b.active) {
|
|
return Number(b.active) - Number(a.active)
|
|
}
|
|
|
|
const aFreshness = a.lastSeenAt ?? a.fetchedAt ?? 0
|
|
const bFreshness = b.lastSeenAt ?? b.fetchedAt ?? 0
|
|
return bFreshness - aFreshness
|
|
})
|
|
|
|
return typeof args.limit === "number" && args.limit > 0
|
|
? normalized.slice(0, args.limit)
|
|
: normalized
|
|
},
|
|
})
|
|
|
|
export const refreshCache = action({
|
|
args: {
|
|
reason: v.optional(v.string()),
|
|
force: v.optional(v.boolean()),
|
|
},
|
|
handler: async (ctx, args) => {
|
|
const now = Date.now()
|
|
const state =
|
|
(await ctx.runQuery(api.ebay.getCacheOverview, {})) || ({
|
|
status: "idle",
|
|
lastSuccessfulAt: null,
|
|
lastAttemptAt: null,
|
|
nextEligibleAt: null,
|
|
lastError: null,
|
|
consecutiveFailures: 0,
|
|
queryCount: 0,
|
|
itemCount: 0,
|
|
sourceQueries: [],
|
|
} as const)
|
|
|
|
if (
|
|
!args.force &&
|
|
typeof state.nextEligibleAt === "number" &&
|
|
state.nextEligibleAt > now
|
|
) {
|
|
return {
|
|
status: "skipped",
|
|
message: "Refresh is deferred until the next eligible window.",
|
|
nextEligibleAt: state.nextEligibleAt,
|
|
}
|
|
}
|
|
|
|
const appId = process.env.EBAY_APP_ID?.trim()
|
|
if (!appId) {
|
|
const nextEligibleAt = now + BASE_REFRESH_MS
|
|
await ctx.runMutation(internal.ebay.upsertPollResult, {
|
|
key: POLL_KEY,
|
|
status: "missing_config",
|
|
lastAttemptAt: now,
|
|
lastSuccessfulAt: state.lastSuccessfulAt || null,
|
|
nextEligibleAt,
|
|
lastError: "EBAY_APP_ID is not configured.",
|
|
consecutiveFailures: state.consecutiveFailures + 1,
|
|
queryCount: 0,
|
|
itemCount: 0,
|
|
sourceQueries: [],
|
|
listings: [],
|
|
reason: args.reason || "missing_config",
|
|
})
|
|
|
|
return {
|
|
status: "missing_config",
|
|
message: "EBAY_APP_ID is not configured.",
|
|
nextEligibleAt,
|
|
}
|
|
}
|
|
|
|
const sourceQueries: string[] = []
|
|
const collectedListings = new Map<string, ReturnType<typeof normalizeEbayItem>>()
|
|
let queryCount = 0
|
|
let rateLimited = false
|
|
let lastError: string | null = null
|
|
|
|
const pollQueries = getPollQueriesForRun(now)
|
|
|
|
for (const query of pollQueries) {
|
|
if (collectedListings.size >= MAX_UNIQUE_RESULTS) {
|
|
break
|
|
}
|
|
|
|
queryCount += 1
|
|
sourceQueries.push(query.label)
|
|
|
|
try {
|
|
const items = await searchEbayListings(query)
|
|
for (const item of items) {
|
|
const listing = normalizeEbayItem(item, now)
|
|
if (!listing.itemId) {
|
|
continue
|
|
}
|
|
|
|
if (!isTrustedListingCandidate(listing)) {
|
|
continue
|
|
}
|
|
|
|
const existing = collectedListings.get(listing.itemId)
|
|
if (existing) {
|
|
existing.sourceQueries = Array.from(
|
|
new Set([...existing.sourceQueries, query.label])
|
|
)
|
|
existing.title = listing.title || existing.title
|
|
existing.normalizedTitle = normalizeText(existing.title)
|
|
existing.price = listing.price || existing.price
|
|
existing.currency = listing.currency || existing.currency
|
|
existing.imageUrl = listing.imageUrl || existing.imageUrl
|
|
existing.viewItemUrl = listing.viewItemUrl || existing.viewItemUrl
|
|
existing.condition = listing.condition || existing.condition
|
|
existing.shippingCost = listing.shippingCost || existing.shippingCost
|
|
existing.affiliateLink = listing.affiliateLink || existing.affiliateLink
|
|
existing.lastSeenAt = now
|
|
existing.fetchedAt = now
|
|
existing.expiresAt = now + LISTING_EXPIRES_MS
|
|
existing.active = true
|
|
continue
|
|
}
|
|
|
|
listing.sourceQueries = [query.label]
|
|
collectedListings.set(listing.itemId, listing)
|
|
}
|
|
} catch (error) {
|
|
lastError = error instanceof Error ? error.message : "Failed to refresh eBay listings"
|
|
if (isRateLimitError(lastError)) {
|
|
rateLimited = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
const listings = Array.from(collectedListings.values())
|
|
const nextEligibleAt = now + getBackoffMs(
|
|
state.consecutiveFailures + 1,
|
|
rateLimited
|
|
)
|
|
|
|
await ctx.runMutation(internal.ebay.upsertPollResult, {
|
|
key: POLL_KEY,
|
|
status: rateLimited
|
|
? "rate_limited"
|
|
: lastError
|
|
? "error"
|
|
: "success",
|
|
lastAttemptAt: now,
|
|
lastSuccessfulAt: rateLimited || lastError ? state.lastSuccessfulAt || null : now,
|
|
nextEligibleAt: rateLimited || lastError ? nextEligibleAt : now + BASE_REFRESH_MS,
|
|
lastError: lastError || null,
|
|
consecutiveFailures:
|
|
rateLimited || lastError ? state.consecutiveFailures + 1 : 0,
|
|
queryCount,
|
|
itemCount: listings.length,
|
|
sourceQueries,
|
|
listings,
|
|
reason: args.reason || "cron",
|
|
})
|
|
|
|
return {
|
|
status: rateLimited ? "rate_limited" : lastError ? "error" : "success",
|
|
message: lastError || undefined,
|
|
queryCount,
|
|
itemCount: listings.length,
|
|
nextEligibleAt: rateLimited || lastError ? nextEligibleAt : now + BASE_REFRESH_MS,
|
|
}
|
|
},
|
|
})
|
|
|
|
export const upsertPollResult = internalMutation({
|
|
args: {
|
|
key: v.string(),
|
|
status: v.union(
|
|
v.literal("idle"),
|
|
v.literal("success"),
|
|
v.literal("rate_limited"),
|
|
v.literal("error"),
|
|
v.literal("missing_config"),
|
|
v.literal("skipped")
|
|
),
|
|
lastAttemptAt: v.number(),
|
|
lastSuccessfulAt: v.union(v.number(), v.null()),
|
|
nextEligibleAt: v.union(v.number(), v.null()),
|
|
lastError: v.union(v.string(), v.null()),
|
|
consecutiveFailures: v.number(),
|
|
queryCount: v.number(),
|
|
itemCount: v.number(),
|
|
sourceQueries: v.array(v.string()),
|
|
listings: v.array(
|
|
v.object({
|
|
itemId: v.string(),
|
|
title: v.string(),
|
|
normalizedTitle: v.string(),
|
|
price: v.string(),
|
|
currency: v.string(),
|
|
imageUrl: v.optional(v.string()),
|
|
viewItemUrl: v.string(),
|
|
condition: v.optional(v.string()),
|
|
shippingCost: v.optional(v.string()),
|
|
affiliateLink: v.string(),
|
|
sourceQueries: v.array(v.string()),
|
|
fetchedAt: v.number(),
|
|
firstSeenAt: v.number(),
|
|
lastSeenAt: v.number(),
|
|
expiresAt: v.number(),
|
|
active: v.boolean(),
|
|
})
|
|
),
|
|
reason: v.optional(v.string()),
|
|
},
|
|
handler: async (ctx, args) => {
|
|
for (const listing of args.listings) {
|
|
const existing = await ctx.db
|
|
.query("ebayListings")
|
|
.withIndex("by_itemId", (q) => q.eq("itemId", listing.itemId))
|
|
.unique()
|
|
|
|
if (existing) {
|
|
await ctx.db.patch(existing._id, {
|
|
title: listing.title,
|
|
normalizedTitle: listing.normalizedTitle,
|
|
price: listing.price,
|
|
currency: listing.currency,
|
|
imageUrl: listing.imageUrl,
|
|
viewItemUrl: listing.viewItemUrl,
|
|
condition: listing.condition,
|
|
shippingCost: listing.shippingCost,
|
|
affiliateLink: listing.affiliateLink,
|
|
sourceQueries: Array.from(
|
|
new Set([...(existing.sourceQueries || []), ...(listing.sourceQueries || [])])
|
|
),
|
|
fetchedAt: listing.fetchedAt,
|
|
firstSeenAt: existing.firstSeenAt || listing.firstSeenAt,
|
|
lastSeenAt: listing.lastSeenAt,
|
|
expiresAt: listing.expiresAt,
|
|
active: listing.active,
|
|
})
|
|
continue
|
|
}
|
|
|
|
await ctx.db.insert("ebayListings", listing)
|
|
}
|
|
|
|
const existingState = await ctx.db
|
|
.query("ebayPollState")
|
|
.withIndex("by_key", (q) => q.eq("key", args.key))
|
|
.unique()
|
|
|
|
const stateRecord: Record<string, any> = {
|
|
key: args.key,
|
|
status: args.status,
|
|
lastAttemptAt: args.lastAttemptAt,
|
|
consecutiveFailures: args.consecutiveFailures,
|
|
queryCount: args.queryCount,
|
|
itemCount: args.itemCount,
|
|
sourceQueries: args.sourceQueries,
|
|
updatedAt: Date.now(),
|
|
}
|
|
|
|
if (args.lastSuccessfulAt !== null) {
|
|
stateRecord.lastSuccessfulAt = args.lastSuccessfulAt
|
|
}
|
|
|
|
if (args.nextEligibleAt !== null) {
|
|
stateRecord.nextEligibleAt = args.nextEligibleAt
|
|
}
|
|
|
|
if (args.lastError !== null) {
|
|
stateRecord.lastError = args.lastError
|
|
}
|
|
|
|
if (existingState) {
|
|
await ctx.db.patch(existingState._id, stateRecord)
|
|
return await ctx.db.get(existingState._id)
|
|
}
|
|
|
|
const id = await ctx.db.insert("ebayPollState", stateRecord)
|
|
return await ctx.db.get(id)
|
|
},
|
|
})
|