Amazon S3

Install

cargo install faucet-cli --features source-s3
faucet init --source s3

# Cargo.toml — use it as a library
cargo add faucet-source-s3

Starter config

source:
  type: s3
  config:
    bucket: ""    # REQUIRED — S3 bucket name.
    # prefix: null    # Object key prefix filter.
    # include: null    # Glob over the whole object key selecting which listed objects to read (e.g.
    # region: null    # AWS region.
    # endpoint_url: null    # Custom endpoint URL for S3-compatible services (e.g.
    # credentials: { type: default }    # one of: default, profile, access_key, assume_role, web_identity, anonymous
      # --- Alternative variants — replace the block above with one of these ---
      # type: profile
      # config:
        # name: ""    # REQUIRED
      #
      # type: access_key
      # config:
        # access_key_id: ""    # REQUIRED
        # secret_access_key: ""    # REQUIRED
        # session_token: ""
      #
      # type: assume_role
      # config:
        # role_arn: ""    # REQUIRED
        # session_name: ""
        # external_id: ""
      #
      # type: web_identity
      #
      # type: anonymous
    # file_format: null    # Format of files stored in S3.
    # max_objects: null    # Maximum number of objects to read.
    # concurrency: 10    # Maximum number of concurrent object reads (default: 10).
    # batch_size: 1000    # Records per emitted `StreamPage`.
    # verify_length: true    # Verify each object's byte length against the length the store advertises, failing the read with `FaucetError::Source` on a short (truncated) or over-long transfer (#161).
    # verify_checksum: false    # Verify each object's body against the checksum the store advertises (#161).
    # max_object_bytes: 2147483648    # Largest object, once decompressed, that a format read whole (JSON array, raw text, CSV, XML, Excel, Avro, ORC, buffered Parquet, and JSON Lines under `verify_checksum`) may reach; a larger one fails with an error instead of exhausting memory.
    # max_decoded_bytes: 0    # Most bytes one object may decode to in memory where a format can expand past its input: an Avro file's records (per batch where the file streams), an Excel workbook's parts and shared strings.
    # compression: null    # User-facing compression config.
    # csv: { ... }    # CSV dialect.
    # excel: { ... }    # Excel worksheet selection.
    # xml: { ... }    # XML record framing.
    # avro: { ... }    # Avro read/write options.
    # orc: { ... }    # ORC read options.
    # parquet: { ... }    # Parquet reader options (the `parquet:` block of an object-store source).

Capabilities

Config fields