Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 03:32:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

finish responding properly

Browse Source
This commit is contained in:
Harsh Gupta 2024-08-15 15:42:39 +05:30
parent 7677ec77ce
commit a7fbe3cb38
1 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1,5 +1,5 @@
import {
assignTransferProtocolMeta, marshalErrorLike,
marshalErrorLike,
RPCHost, RPCReflection,
HashManager,
AssertionFailureError, ParamValidationError, Defer,
@ -28,6 +28,27 @@ const md5Hasher = new HashManager('md5', 'hex');
// const logger = new Logger('Crawler');
import { TransferProtocolMetadata } from 'civkit';
function sendResponse<T>(res: Response, data: T, meta: TransferProtocolMetadata): T {
if (meta.code) {
res.status(meta.code);
}
if (meta.contentType) {
res.type(meta.contentType);
}
if (meta.headers) {
for (const [key, value] of Object.entries(meta.headers)) {
if (value !== undefined) {
res.setHeader(key, value);
}
}
}
res.send(data);
return data;
}
export interface ExtraScrappingOptions extends ScrappingOptions {
withIframe?: boolean;
targetSelector?: string | string[];
@ -596,6 +617,8 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
async crawl(req: Request, res: Response) {
console.log('Crawl method called with request:', req.url);
// res.setHeader('Access-Control-Allow-Origin', '*');
// res.send('Helloooooooo!');
// const rpcReflect: RPCReflection = {};
const ctx = { req, res };
console.log(`req.headers: ${JSON.stringify(req.headers)}`);
@ -610,7 +633,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return this.getIndex();
}
return assignTransferProtocolMeta(`${this.getIndex()}`,
return sendResponse(res, `${this.getIndex()}`,
{ contentType: 'text/plain', envelope: null }
);
}
@ -717,17 +740,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
if (crawlerOptions.timeout === undefined) {
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`,
return sendResponse(res, `${formatted}`,
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }
);
}
if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`,
return sendResponse(res, `${formatted}`,
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } }
);
}
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
return sendResponse(res, `${formatted}`, { contentType: 'text/plain', envelope: null });
}
}
@ -737,17 +760,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, lastScrapped, urlToCrawl);
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`,
return sendResponse(res, `${formatted}`,
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }
);
}
if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`,
return sendResponse(res, `${formatted}`,
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } }
);
}
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
return sendResponse(res, `${formatted}`, { contentType: 'text/plain', envelope: null });
}
getUrlDigest(urlToCrawl: URL) {