REST

Install

cargo install faucet-cli --features source-rest
faucet init --source rest

# Cargo.toml — use it as a library
cargo add faucet-source-rest

Starter config

source:
  type: rest
  config:
    base_url: ""    # REQUIRED — Base URL of the API, e.g.
    # path: ""    # URL path, relative to `base_url`.
    # method: "GET"    # HTTP method for the data request.
    # auth: { type: none }    # one of: none, bearer, basic, api_key, api_key_query, oauth2, token_endpoint, custom
      # --- Alternative variants — replace the block above with one of these ---
      # type: bearer
      # config:
        # token: ""    # REQUIRED
      #
      # type: basic
      # config:
        # username: ""    # REQUIRED
        # password: ""    # REQUIRED
      #
      # type: api_key
      # config:
        # header: ""    # REQUIRED
        # value: ""    # REQUIRED
      #
      # type: api_key_query
      # config:
        # param: ""    # REQUIRED
        # value: ""    # REQUIRED
      #
      # type: oauth2
      # config:
        # token_url: ""    # REQUIRED
        # client_id: ""    # REQUIRED
        # client_secret: ""    # REQUIRED
        # scopes: []    # REQUIRED
        # expiry_ratio: 0.9
      #
      # type: token_endpoint
      # config:
        # url: ""    # REQUIRED
        # method: ""    # REQUIRED
        # body: null
        # token_path: ""    # REQUIRED
        # expiry_path: ""
        # expiry_ratio: 0.9
        # encoding: null
      #
      # type: custom
      # config:
        # headers: {}    # REQUIRED
    # headers: {}    # Static request headers sent on **every** request (data pages, async-job submit/poll/fetch requests, and OData `$metadata` discovery probes).
    # trusted_hosts: []    # Extra hosts that may receive this source's credentials when a server-given URL (a next-page link, an async-job URL, a discovery listing's next link) points at them, or a redirect leads to them.
    # query_params: {}    # Static query-string parameters, rendered as `?k=v`.
    # query_params_multi: {}    # Repeated / array-valued query params (#536), rendered as repeated keys — e.g.
    # body: null    # JSON request body, for `POST`/`PUT`-style reads.
    # pagination: { type: None }    # one of: None, Cursor, CursorInBody, LinkHeader, NextLinkInBody, PageNumber, Offset, OffsetInBody, RecordFieldCursor
      # --- Alternative variants — replace the block above with one of these ---
      # type: Cursor
      # next_token_path: ""    # REQUIRED
      # param_name: ""    # REQUIRED
      #
      # type: CursorInBody
      # next_token_path: ""    # REQUIRED
      # body_cursor_field: ""    # REQUIRED
      #
      # type: LinkHeader
      #
      # type: NextLinkInBody
      # next_link_path: ""    # REQUIRED
      #
      # type: PageNumber
      # param_name: ""    # REQUIRED
      # start_page: 0    # REQUIRED
      # page_size: 0
      # page_size_param: ""
      #
      # type: Offset
      # offset_param: ""    # REQUIRED
      # limit_param: ""    # REQUIRED
      # limit: 0    # REQUIRED
      # total_path: ""
      # rows_path: ""
      # stop_when_short: false
      #
      # type: OffsetInBody
      # offset_field: ""    # REQUIRED
      # limit_field: ""    # REQUIRED
      # limit: 0    # REQUIRED
      # stop_when_short: false
      # rows_path: ""
      # total_path: ""
      #
      # type: RecordFieldCursor
      # field: ""    # REQUIRED
      # into: null
      # param: ""    # REQUIRED
      # agg: null
      # stop_when_short: true
      # page_size: 0    # REQUIRED
      # inclusive: false
    # records_path: null    # JSONPath to the array of records inside the response body, e.g.
    # allow_missing_records_path: false    # Accept a response in which `records_path` resolves to nothing (the key is absent) as an empty page.
    # max_pages: null    # Optional cap on pages fetched per pass, across every pagination style.
    # allow_repeated_pages: false    # Keep paging when a `PageNumber` / `Offset` / `OffsetInBody` page holds the same records as an earlier, non-adjacent page of the pass, instead of failing the run.
    # assume_ordered: false    # The feed returns records in ascending `replication_key` order, so an incremental pass that `max_pages` cuts short may persist the largest key it read and resume from it next run.
    # request_delay: null    # Fixed delay between page requests, in seconds — a politeness knob for APIs that rate-limit by request frequency.
    # timeout: 30    # Per-request timeout in seconds.
    # max_retries: 3    # Number of retries (after the first attempt) for transient request failures.
    # retry_backoff: 1    # Base exponential-backoff delay between retries.
    # tolerated_http_errors: []    # HTTP status codes that should **not** cause an error.
    # retry_on_response: []    # Responses to treat as throttling and retry, for APIs that signal a rate limit with a 4xx other than 429 plus an error code in the body (e.g.
    # replication_method: { type: FullTable }    # one of: FullTable, Incremental
      # --- Alternative variants — replace the block above with one of these ---
      # type: Incremental
    # replication_key: null    # Field used for incremental replication bookmarking: a top-level field name (`updated_at`), a dot path into nested objects (`fields.updated`; numeric segments index arrays, and a literal top-level field of that exact name wins), or a JSON Pointer (`/fields/updated`) for field names that contain dots.
    # on_missing_key: null    # What to do with a record whose replication key is missing or `null` (#747).
    # start_replication_value: null    # Bookmark value: records where `record[replication_key] <= start_replication_value` are filtered out when `replication_me…
    # state_key: null    # Opt-in identifier used by `Pipeline::with_state_store` to persist this stream's bookmark across runs.
    # name: null    # Human-readable stream name (used in logging and Singer SCHEMA messages).
    # primary_keys: []    # Field names that uniquely identify a record (Singer `key_properties`).
    # schema: null    # JSON Schema describing the structure of each record.
    # schema_sample_size: 100    # Maximum number of records to sample when inferring the schema via `crate::stream::RestStream::infer_schema`.
    # requests: []    # `requests:` — one entry per request: a context map whose values are substituted into `path` placeholders.
    # request_concurrency: null    # `request_concurrency:` — maximum number of `requests` entries fetched concurrently (`partition_concurrency:` is still accepted).
    # tls: null    # Optional client-certificate (mutual TLS) config.
    # response_format: null    # How to parse the response body into records.
    # csv_delimiter: 44    # CSV field delimiter byte (default `,`).
    # csv_has_headers: true    # Whether the first CSV row is a header row supplying field names (default `true`).
    # csv_quote: 34    # CSV quote character (default `"`).
    # csv_null_values: []    # CSV fields equal to any of these strings decode as JSON `null` instead of a string (default: none).
    # excel_sheet: null    # Excel worksheet to read: a sheet name, or a 0-based index as a string.
    # excel_header_row: 0    # 0-based index of the Excel header row (default `0`).
    # replication_bind: null    # Bind the stored bookmark into the outgoing request (query param / header / body field / path) so the server returns only new rows.
    # odata: null    # Speak the OData protocol: `@odata.nextLink` paging, the `$.value` envelope, `$select`/`$filter`/`$expand`/`$orderby` sugar, and `$metadata` (EDMX) → schema discovery.
    # drop_key_prefixes: []    # Drop per-record keys starting with any of these prefixes (#654 M24).
    # decode: []    # Decode the response body before record extraction: a chain of `extract` (JSONPath) / `base64` / `gunzip` / `unzip` / `parse` (json|csv|xlsx|xml) steps.
    # async_job: null    # Run a submit→poll→fetch job lifecycle instead of a single GET, for bulk/export/report-run APIs (bulk export jobs, report runs, …).
    # window: null    # Bound each request to a rolling `[start, end)` window between the stored bookmark and `now`, iterating the windows within one run (each `step` wide) with per-window bookmark durability.
    # record_ancestors: {}    # When `records_path` selects a **nested** array element (e.g.
    # records_multi: []    # Emit several record arrays from one response in a single page (sharing one pagination advance), each stamped with a user-defined op marker under `op_field`.
    # op_field: ""    # Field name each `records_multi` record is stamped with its spec's `op` value.
    # records_route: null    # Route each record to a named stream by its GID type or a discriminator field, stamping the stream into `stream_field` — so one bulk job's parents and children reach separate sinks.
    # persist_cursor: false    # Persist the terminal pagination cursor as this run's bookmark (riding the existing `StreamPage.bookmark` / `StateStore` path — no core trait change) and, on resume, seed the stored bookmark back into the first request (query param for `cursor`, request body field for `cursor_in_body`) before paging.
    # discovery: null    # Generic, vendor-neutral discovery recipe: declare which API calls to make and how to extract datasets from their JSON responses (reusing this source's `auth` / `base_url` / `headers`).

Capabilities

Config fields