web

web / lib/constants/ingest-constants

lib/constants/ingest-constants

Defines constants used throughout the data ingest processing system.

This file centralizes constant values to prevent string duplication and provide a single source of truth for statuses, stages, job types, and collection names related to the ingest pipeline. This improves maintainability and reduces the risk of typos.

Batch sizes are loaded from config/timetiles.yml (if present) with env var overrides and hardcoded defaults as fallback. See getAppConfig for details.

Type Aliases

ProcessingStage

ProcessingStage = typeof PROCESSING_STAGE[keyof typeof PROCESSING_STAGE]

JobType

JobType = typeof JOB_TYPES[keyof typeof JOB_TYPES]

CollectionName

CollectionName = typeof COLLECTION_NAMES[keyof typeof COLLECTION_NAMES]

Variables

PROCESSING_STAGE

const PROCESSING_STAGE: object

Constants for ingest processing to avoid string duplication.

Type Declaration

ANALYZE_DUPLICATES

readonly ANALYZE_DUPLICATES: "analyze-duplicates" = "analyze-duplicates"

DETECT_SCHEMA

readonly DETECT_SCHEMA: "detect-schema" = "detect-schema"

VALIDATE_SCHEMA

readonly VALIDATE_SCHEMA: "validate-schema" = "validate-schema"

NEEDS_REVIEW

readonly NEEDS_REVIEW: "needs-review" = "needs-review"

CREATE_SCHEMA_VERSION

readonly CREATE_SCHEMA_VERSION: "create-schema-version" = "create-schema-version"

GEOCODE_BATCH

readonly GEOCODE_BATCH: "geocode-batch" = "geocode-batch"

CREATE_EVENTS

readonly CREATE_EVENTS: "create-events" = "create-events"

COMPLETED

readonly COMPLETED: "completed" = "completed"

FAILED

readonly FAILED: "failed" = "failed"

JOB_TYPES

const JOB_TYPES: object

Type Declaration

DATASET_DETECTION

readonly DATASET_DETECTION: "dataset-detection" = "dataset-detection"

ANALYZE_DUPLICATES

readonly ANALYZE_DUPLICATES: "analyze-duplicates" = "analyze-duplicates"

DETECT_SCHEMA

readonly DETECT_SCHEMA: "detect-schema" = "detect-schema"

VALIDATE_SCHEMA

readonly VALIDATE_SCHEMA: "validate-schema" = "validate-schema"

CREATE_SCHEMA_VERSION

readonly CREATE_SCHEMA_VERSION: "create-schema-version" = "create-schema-version"

GEOCODE_BATCH

readonly GEOCODE_BATCH: "geocode-batch" = "geocode-batch"

CREATE_EVENTS

readonly CREATE_EVENTS: "create-events" = "create-events"

URL_FETCH

readonly URL_FETCH: "url-fetch" = "url-fetch"

SCHEDULE_MANAGER

readonly SCHEDULE_MANAGER: "schedule-manager" = "schedule-manager"

CACHE_CLEANUP

readonly CACHE_CLEANUP: "cache-cleanup" = "cache-cleanup"

COLLECTION_NAMES

const COLLECTION_NAMES: object

Type Declaration

INGEST_FILES

readonly INGEST_FILES: "ingest-files" = "ingest-files"

INGEST_JOBS

readonly INGEST_JOBS: "ingest-jobs" = "ingest-jobs"

EVENTS

readonly EVENTS: "events" = "events"

CATALOGS

readonly CATALOGS: "catalogs" = "catalogs"

DATASETS

readonly DATASETS: "datasets" = "datasets"

DATASET_SCHEMAS

readonly DATASET_SCHEMAS: "dataset-schemas" = "dataset-schemas"

GEOCODING_PROVIDERS

readonly GEOCODING_PROVIDERS: "geocoding-providers" = "geocoding-providers"

SCHEDULED_INGESTS

readonly SCHEDULED_INGESTS: "scheduled-ingests" = "scheduled-ingests"

USERS

readonly USERS: "users" = "users"

PAYLOAD_MIGRATIONS

readonly PAYLOAD_MIGRATIONS: "payload-migrations" = "payload-migrations"

BATCH_SIZES

const BATCH_SIZES: object

Type Declaration

DUPLICATE_ANALYSIS

readonly DUPLICATE_ANALYSIS: number = appBatchSizes.duplicateAnalysis

SCHEMA_DETECTION

readonly SCHEMA_DETECTION: number = appBatchSizes.schemaDetection

EVENT_CREATION

readonly EVENT_CREATION: number = appBatchSizes.eventCreation

DATABASE_CHUNK

readonly DATABASE_CHUNK: number = appBatchSizes.databaseChunk