DMleadgen
46d973904b
Next.js website for Rocky Mountain Vending company featuring: - Product catalog with Stripe integration - Service areas and parts pages - Admin dashboard with Clerk authentication - SEO optimized pages with JSON-LD structured data Co-authored-by: Cursor <cursoragent@cursor.com>
345 lines
No EOL
14 KiB
Text
345 lines
No EOL
14 KiB
Text
import { InvalidArgumentException, NoSuchInterceptException, NoSuchNetworkDataException, UnsupportedOperationException, } from '../../../protocol/protocol.js';
|
|
import { uuidv4 } from '../../../utils/uuid.js';
|
|
import { CollectorsStorage } from './CollectorsStorage.js';
|
|
import { NetworkRequest } from './NetworkRequest.js';
|
|
import { matchUrlPattern } from './NetworkUtils.js';
|
|
// The default total data size limit in CDP.
|
|
// https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/inspector/inspector_network_agent.cc;drc=da1f749634c9a401cc756f36c2e6ce233e1c9b4d;l=133
|
|
export const MAX_TOTAL_COLLECTED_SIZE = 200_000_000;
|
|
/** Stores network and intercept maps. */
|
|
export class NetworkStorage {
|
|
#browsingContextStorage;
|
|
#eventManager;
|
|
#collectorsStorage;
|
|
#logger;
|
|
/**
|
|
* A map from network request ID to Network Request objects.
|
|
* Needed as long as information about requests comes from different events.
|
|
*/
|
|
#requests = new Map();
|
|
/** A map from intercept ID to track active network intercepts. */
|
|
#intercepts = new Map();
|
|
#defaultCacheBehavior = 'default';
|
|
constructor(eventManager, browsingContextStorage, browserClient, logger) {
|
|
this.#browsingContextStorage = browsingContextStorage;
|
|
this.#eventManager = eventManager;
|
|
this.#collectorsStorage = new CollectorsStorage(MAX_TOTAL_COLLECTED_SIZE, logger);
|
|
browserClient.on('Target.detachedFromTarget', ({ sessionId }) => {
|
|
this.disposeRequestMap(sessionId);
|
|
});
|
|
this.#logger = logger;
|
|
}
|
|
/**
|
|
* Gets the network request with the given ID, if any.
|
|
* Otherwise, creates a new network request with the given ID and cdp target.
|
|
*/
|
|
#getOrCreateNetworkRequest(id, cdpTarget, redirectCount) {
|
|
let request = this.getRequestById(id);
|
|
if (redirectCount === undefined && request) {
|
|
// Force re-creating requests for redirects.
|
|
return request;
|
|
}
|
|
request = new NetworkRequest(id, this.#eventManager, this, cdpTarget, redirectCount, this.#logger);
|
|
this.addRequest(request);
|
|
return request;
|
|
}
|
|
onCdpTargetCreated(cdpTarget) {
|
|
const cdpClient = cdpTarget.cdpClient;
|
|
// TODO: Wrap into object
|
|
const listeners = [
|
|
[
|
|
'Network.requestWillBeSent',
|
|
(params) => {
|
|
const request = this.getRequestById(params.requestId);
|
|
request?.updateCdpTarget(cdpTarget);
|
|
if (request && request.isRedirecting()) {
|
|
request.handleRedirect(params);
|
|
this.disposeRequest(params.requestId);
|
|
this.#getOrCreateNetworkRequest(params.requestId, cdpTarget, request.redirectCount + 1).onRequestWillBeSentEvent(params);
|
|
}
|
|
else {
|
|
this.#getOrCreateNetworkRequest(params.requestId, cdpTarget).onRequestWillBeSentEvent(params);
|
|
}
|
|
},
|
|
],
|
|
[
|
|
'Network.requestWillBeSentExtraInfo',
|
|
(params) => {
|
|
const request = this.#getOrCreateNetworkRequest(params.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onRequestWillBeSentExtraInfoEvent(params);
|
|
},
|
|
],
|
|
[
|
|
'Network.responseReceived',
|
|
(params) => {
|
|
const request = this.#getOrCreateNetworkRequest(params.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onResponseReceivedEvent(params);
|
|
},
|
|
],
|
|
[
|
|
'Network.responseReceivedExtraInfo',
|
|
(params) => {
|
|
const request = this.#getOrCreateNetworkRequest(params.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onResponseReceivedExtraInfoEvent(params);
|
|
},
|
|
],
|
|
[
|
|
'Network.requestServedFromCache',
|
|
(params) => {
|
|
const request = this.#getOrCreateNetworkRequest(params.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onServedFromCache();
|
|
},
|
|
],
|
|
[
|
|
'Network.loadingFailed',
|
|
(params) => {
|
|
const request = this.#getOrCreateNetworkRequest(params.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onLoadingFailedEvent(params);
|
|
},
|
|
],
|
|
[
|
|
'Fetch.requestPaused',
|
|
(event) => {
|
|
const request = this.#getOrCreateNetworkRequest(
|
|
// CDP quirk if the Network domain is not present this is undefined
|
|
event.networkId ?? event.requestId, cdpTarget);
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onRequestPaused(event);
|
|
},
|
|
],
|
|
[
|
|
'Fetch.authRequired',
|
|
(event) => {
|
|
let request = this.getRequestByFetchId(event.requestId);
|
|
if (!request) {
|
|
request = this.#getOrCreateNetworkRequest(event.requestId, cdpTarget);
|
|
}
|
|
request.updateCdpTarget(cdpTarget);
|
|
request.onAuthRequired(event);
|
|
},
|
|
],
|
|
[
|
|
'Network.dataReceived',
|
|
(params) => {
|
|
this.getRequestById(params.requestId)?.updateCdpTarget(cdpTarget);
|
|
},
|
|
],
|
|
[
|
|
'Network.loadingFinished',
|
|
(params) => {
|
|
this.getRequestById(params.requestId)?.updateCdpTarget(cdpTarget);
|
|
},
|
|
],
|
|
];
|
|
for (const [event, listener] of listeners) {
|
|
cdpClient.on(event, listener);
|
|
}
|
|
}
|
|
async getCollectedData(params) {
|
|
if (!this.#collectorsStorage.isCollected(params.request, params.dataType, params.collector)) {
|
|
throw new NoSuchNetworkDataException(params.collector === undefined
|
|
? `No collected ${params.dataType} data`
|
|
: `Collector ${params.collector} didn't collect ${params.dataType} data`);
|
|
}
|
|
if (params.disown && params.collector === undefined) {
|
|
throw new InvalidArgumentException('Cannot disown collected data without collector ID');
|
|
}
|
|
const request = this.getRequestById(params.request);
|
|
if (request === undefined) {
|
|
throw new NoSuchNetworkDataException(`No data for ${params.request}`);
|
|
}
|
|
let result = undefined;
|
|
switch (params.dataType) {
|
|
case "response" /* Network.DataType.Response */:
|
|
result = await this.#getCollectedResponseData(request);
|
|
break;
|
|
case "request" /* Network.DataType.Request */:
|
|
result = await this.#getCollectedRequestData(request);
|
|
break;
|
|
default:
|
|
throw new UnsupportedOperationException(`Unsupported data type ${params.dataType}`);
|
|
}
|
|
if (params.disown && params.collector !== undefined) {
|
|
this.#collectorsStorage.disownData(request.id, params.dataType, params.collector);
|
|
// `disposeRequest` disposes request only if no other collectors for it are left.
|
|
this.disposeRequest(request.id);
|
|
}
|
|
return result;
|
|
}
|
|
async #getCollectedResponseData(request) {
|
|
try {
|
|
const responseBody = await request.cdpClient.sendCommand('Network.getResponseBody', { requestId: request.id });
|
|
return {
|
|
bytes: {
|
|
type: responseBody.base64Encoded ? 'base64' : 'string',
|
|
value: responseBody.body,
|
|
},
|
|
};
|
|
}
|
|
catch (error) {
|
|
if (error.code === -32000 /* CdpErrorConstants.GENERIC_ERROR */ &&
|
|
error.message === 'No resource with given identifier found') {
|
|
// The data has be gone for whatever reason.
|
|
throw new NoSuchNetworkDataException(`Response data was disposed`);
|
|
}
|
|
if (error.code === -32001 /* CdpErrorConstants.CONNECTION_CLOSED */) {
|
|
// The request's CDP session is gone. http://b/450771615.
|
|
throw new NoSuchNetworkDataException(`Response data is disposed after the related page`);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
async #getCollectedRequestData(request) {
|
|
// TODO: handle CDP error in case of the renderer is gone.
|
|
const requestPostData = await request.cdpClient.sendCommand('Network.getRequestPostData', { requestId: request.id });
|
|
return {
|
|
bytes: {
|
|
type: 'string',
|
|
value: requestPostData.postData,
|
|
},
|
|
};
|
|
}
|
|
collectIfNeeded(request, dataType) {
|
|
this.#collectorsStorage.collectIfNeeded(request, dataType, request.cdpTarget.topLevelId, request.cdpTarget.userContext);
|
|
}
|
|
getInterceptionStages(browsingContextId) {
|
|
const stages = {
|
|
request: false,
|
|
response: false,
|
|
auth: false,
|
|
};
|
|
for (const intercept of this.#intercepts.values()) {
|
|
if (intercept.contexts &&
|
|
!intercept.contexts.includes(browsingContextId)) {
|
|
continue;
|
|
}
|
|
stages.request ||= intercept.phases.includes("beforeRequestSent" /* Network.InterceptPhase.BeforeRequestSent */);
|
|
stages.response ||= intercept.phases.includes("responseStarted" /* Network.InterceptPhase.ResponseStarted */);
|
|
stages.auth ||= intercept.phases.includes("authRequired" /* Network.InterceptPhase.AuthRequired */);
|
|
}
|
|
return stages;
|
|
}
|
|
getInterceptsForPhase(request, phase) {
|
|
if (request.url === NetworkRequest.unknownParameter) {
|
|
return new Set();
|
|
}
|
|
const intercepts = new Set();
|
|
for (const [interceptId, intercept] of this.#intercepts.entries()) {
|
|
if (!intercept.phases.includes(phase) ||
|
|
(intercept.contexts &&
|
|
!intercept.contexts.includes(request.cdpTarget.topLevelId))) {
|
|
continue;
|
|
}
|
|
if (intercept.urlPatterns.length === 0) {
|
|
intercepts.add(interceptId);
|
|
continue;
|
|
}
|
|
for (const pattern of intercept.urlPatterns) {
|
|
if (matchUrlPattern(pattern, request.url)) {
|
|
intercepts.add(interceptId);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return intercepts;
|
|
}
|
|
disposeRequestMap(sessionId) {
|
|
for (const request of this.#requests.values()) {
|
|
if (request.cdpClient.sessionId === sessionId) {
|
|
this.#requests.delete(request.id);
|
|
request.dispose();
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* Adds the given entry to the intercept map.
|
|
* URL patterns are assumed to be parsed.
|
|
*
|
|
* @return The intercept ID.
|
|
*/
|
|
addIntercept(value) {
|
|
const interceptId = uuidv4();
|
|
this.#intercepts.set(interceptId, value);
|
|
return interceptId;
|
|
}
|
|
/**
|
|
* Removes the given intercept from the intercept map.
|
|
* Throws NoSuchInterceptException if the intercept does not exist.
|
|
*/
|
|
removeIntercept(intercept) {
|
|
if (!this.#intercepts.has(intercept)) {
|
|
throw new NoSuchInterceptException(`Intercept '${intercept}' does not exist.`);
|
|
}
|
|
this.#intercepts.delete(intercept);
|
|
}
|
|
getRequestsByTarget(target) {
|
|
const requests = [];
|
|
for (const request of this.#requests.values()) {
|
|
if (request.cdpTarget === target) {
|
|
requests.push(request);
|
|
}
|
|
}
|
|
return requests;
|
|
}
|
|
getRequestById(id) {
|
|
return this.#requests.get(id);
|
|
}
|
|
getRequestByFetchId(fetchId) {
|
|
for (const request of this.#requests.values()) {
|
|
if (request.fetchId === fetchId) {
|
|
return request;
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
addRequest(request) {
|
|
this.#requests.set(request.id, request);
|
|
}
|
|
/**
|
|
* Disposes the given request, if no collectors targeting it are left.
|
|
*/
|
|
disposeRequest(id) {
|
|
if (this.#collectorsStorage.isCollected(id)) {
|
|
// Keep request, as it's data can be accessed later.
|
|
return;
|
|
}
|
|
// TODO: dispose Network data from Chromium once there is a CDP command for that.
|
|
this.#requests.delete(id);
|
|
}
|
|
/**
|
|
* Gets the virtual navigation ID for the given navigable ID.
|
|
*/
|
|
getNavigationId(contextId) {
|
|
if (contextId === undefined) {
|
|
return null;
|
|
}
|
|
return (this.#browsingContextStorage.findContext(contextId)?.navigationId ?? null);
|
|
}
|
|
set defaultCacheBehavior(behavior) {
|
|
this.#defaultCacheBehavior = behavior;
|
|
}
|
|
get defaultCacheBehavior() {
|
|
return this.#defaultCacheBehavior;
|
|
}
|
|
addDataCollector(params) {
|
|
return this.#collectorsStorage.addDataCollector(params);
|
|
}
|
|
removeDataCollector(params) {
|
|
const releasedRequests = this.#collectorsStorage.removeDataCollector(params.collector);
|
|
releasedRequests.map((request) => this.disposeRequest(request));
|
|
}
|
|
disownData(params) {
|
|
if (!this.#collectorsStorage.isCollected(params.request, params.dataType, params.collector)) {
|
|
throw new NoSuchNetworkDataException(`Collector ${params.collector} didn't collect ${params.dataType} data`);
|
|
}
|
|
this.#collectorsStorage.disownData(params.request, params.dataType, params.collector);
|
|
// `disposeRequest` disposes request only if no other collectors for it are left.
|
|
this.disposeRequest(params.request);
|
|
}
|
|
}
|
|
//# sourceMappingURL=NetworkStorage.js.map |