web / lib/jobs/handlers/url-fetch-job

lib/jobs/handlers/url-fetch-job

URL Fetch Job Handler.

This job handles fetching data from URLs for URL-based and scheduled imports. It downloads the data, saves it to the file system, and updates the import-files record to trigger the existing dataset-detection pipeline.

Interfaces

UrlFetchJobInput

Properties

scheduledImportId?

optional scheduledImportId: string

sourceUrl

sourceUrl: string

authConfig?

optional authConfig: object

type?

optional type: "none" | "api-key" | "bearer" | "basic" | null

apiKey?

optional apiKey: string | null

API key to include in request header

apiKeyHeader?

optional apiKeyHeader: string | null

Header name for API key

bearerToken?

optional bearerToken: string | null

Bearer token for Authorization header

username?

optional username: string | null

Username for basic authentication

password?

optional password: string | null

Password for basic authentication

customHeaders?

optional customHeaders: string | number | boolean | unknown[] | { \[k: string\]: unknown; } | null

Additional custom headers as JSON object

catalogId?

optional catalogId: string | number

originalName

originalName: string

userId?

optional userId: string | number

triggeredBy?

optional triggeredBy: "schedule" | "webhook" | "manual"

Variables

urlFetchJob

const urlFetchJob: object

Type Declaration

slug

slug: string = "url-fetch"

handler()

handler: (context) => Promise<{ output: \{ success: boolean; importFileId: string \| number; filename: string; contentHash: string; isDuplicate: boolean; contentType: string; fileSize: number \| undefined; skippedReason?: string; }; } | { output: \{ success: boolean; error: string; errorDetails: \{ name: string; stack: string \| undefined; }; }; } | { output: \{ success: boolean; error: string; }; }>

Parameters

context

JobHandlerContext

Returns

Promise<{ output: \{ success: boolean; importFileId: string \| number; filename: string; contentHash: string; isDuplicate: boolean; contentType: string; fileSize: number \| undefined; skippedReason?: string; }; } | { output: \{ success: boolean; error: string; errorDetails: \{ name: string; stack: string \| undefined; }; }; } | { output: \{ success: boolean; error: string; }; }>