web / lib/jobs/handlers/url-fetch-job

lib/jobs/handlers/url-fetch-job

URL Fetch Job Handler.

This job handles fetching data from URLs for URL-based and scheduled imports. It downloads the data, saves it to the file system, and updates the import-files record to trigger the existing dataset-detection pipeline.

Interfaces

UrlFetchJobInput

Properties

scheduledImportId?

optional scheduledImportId: string

sourceUrl

sourceUrl: string

authConfig?

optional authConfig: object

type?

optional type: null | "none" | "api-key" | "bearer" | "basic"

apiKey?

optional apiKey: null | string

API key to include in request header

apiKeyHeader?

optional apiKeyHeader: null | string

Header name for API key

bearerToken?

optional bearerToken: null | string

Bearer token for Authorization header

username?

optional username: null | string

Username for basic authentication

password?

optional password: null | string

Password for basic authentication

customHeaders?

optional customHeaders: null | string | number | boolean | unknown[] | {[k: string]: unknown; }

Additional custom headers as JSON object

catalogId?

optional catalogId: string | number

originalName

originalName: string

userId?

optional userId: string | number

triggeredBy?

optional triggeredBy: "schedule" | "webhook" | "manual"

Variables

urlFetchJob

const urlFetchJob: object

Type declaration

slug

slug: string = "url-fetch"

handler()

handler: (context) => Promise<{ output: { success: boolean; importFileId: string | number; filename: string; contentHash: string; isDuplicate: boolean; contentType: string; fileSize: undefined | number; skippedReason?: string; }; } | { output: { success: boolean; error: string; errorDetails: { name: string; stack: undefined | string; }; }; }>

Parameters

context

JobHandlerContext

Returns

Promise<{ output: { success: boolean; importFileId: string | number; filename: string; contentHash: string; isDuplicate: boolean; contentType: string; fileSize: undefined | number; skippedReason?: string; }; } | { output: { success: boolean; error: string; errorDetails: { name: string; stack: undefined | string; }; }; }>