Local files

Install

cargo install faucet-cli --features source-file
faucet init --source file

# Cargo.toml — use it as a library
cargo add faucet-source-file

Starter config

source:
  type: file
  config:
    path: ""    # REQUIRED — A file, a directory, a glob (`data/**/*.jsonl`), or one `http://` / `https://` URL.
    # format: null    # Which format a file is in: resolved from its extension (`auto`) or fixed.
    # compression: null    # User-facing compression config.
    # recursive: false    # Descend into subdirectories when `path` is a directory.
    # concurrency: 4    # Files opened ahead of the one being decoded (ordered prefetch).
    # batch_size: 1000    # Records per emitted page.
    # incremental: null    # Read only files that are new since the previous run.
    # stable_for_secs: 0    # Skip files modified less than this many seconds ago — files still being written.
    # strict: false    # Refuse (instead of skipping with a warning) a file whose extension names no format under `format: auto`.
    # max_object_bytes: 2147483648    # Largest file, once decompressed, that a format read whole (JSON array, XML, Excel, compressed or remote Avro/ORC/Parquet, encrypted files) may reach; a larger one fails with an error instead of exhausting memory.
    # max_decoded_bytes: 0    # Most bytes one object may decode to in memory where a format can expand past its input: an Avro file's records (per batch where the file streams), an Excel workbook's parts and shared strings.
    # max_files: 0    # Read at most this many files (after sorting and filtering).
    # headers: {}    # Request headers for an `http(s)://` path (e.g.
    # http_retries: 3    # Retries for a failed HTTP request (connection error, 429, 5xx), with exponential backoff that honours `Retry-After`.
    # http_connect_timeout_secs: 30    # Seconds to wait for an HTTP connection to open.
    # http_read_timeout_secs: 300    # Seconds an HTTP response may go without sending a byte — waiting for the headers or mid-body.
    # csv: { ... }    # CSV dialect.
    # excel: { ... }    # Excel worksheet selection.
    # xml: { ... }    # XML record framing.
    # avro: { ... }    # Avro read/write options.
    # orc: { ... }    # ORC read options.
    # parquet: { ... }    # Parquet reader options (the `parquet:` block of an object-store source).
    # encryption: null    # Decrypt files sealed by the file or jsonl sink's `encryption` block (AES-256-GCM).

Capabilities

Config fields