From 714dce0392487ab4e4fc86fa55776d228e7fbed7 Mon Sep 17 00:00:00 2001 From: Hofi Date: Thu, 27 Nov 2025 12:38:15 +0100 Subject: [PATCH 1/6] ci: harden the concurrency group name Signed-off-by: Hofi --- .github/workflows/jekyll-gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml index 6ba388c8..a62856f6 100644 --- a/.github/workflows/jekyll-gh-pages.yml +++ b/.github/workflows/jekyll-gh-pages.yml @@ -23,7 +23,7 @@ permissions: # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # Allow canceling of the in-progress deployment now. concurrency: - group: "pages" + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: From eee62617f5156bf7a42d65832768804139062ebc Mon Sep 17 00:00:00 2001 From: Hofi Date: Tue, 25 Nov 2025 15:40:16 +0100 Subject: [PATCH 2/6] admin-guide: further deprecation adjustments Signed-off-by: Hofi --- .../doc/admin-guide/options/deprecated-options.md | 2 +- .../doc/admin-guide/options/log-fetch-delays.md | 4 ++-- _includes/doc/admin-guide/options/log-prefix.md | 12 ++---------- .../070_Destinations/130_MongoDB/README.md | 2 -- .../090_Global_options/000_Global_options.md | 12 ++++++------ .../affile/file-source-driver.md | 2 +- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/_includes/doc/admin-guide/options/deprecated-options.md b/_includes/doc/admin-guide/options/deprecated-options.md index 56dd5e73..3c48e4ea 100644 --- a/_includes/doc/admin-guide/options/deprecated-options.md +++ b/_includes/doc/admin-guide/options/deprecated-options.md @@ -1,2 +1,2 @@ -**NOTE:** The deprecated `{{ include.old }}` option is an alias for the `{{ include.new }}` option, retained for compatibility with earlier {{ site.product.short_name }} versions, but it may be removed at any time without further notice! +**NOTE:** The deprecated `{{ include.old }}` option is an alias for the {{ include.new }} option, retained for compatibility with earlier {{ site.product.short_name }} versions, but it may be removed at any time without further notice! {: .notice--info} diff --git a/_includes/doc/admin-guide/options/log-fetch-delays.md b/_includes/doc/admin-guide/options/log-fetch-delays.md index 06aa2183..f9f2f11f 100644 --- a/_includes/doc/admin-guide/options/log-fetch-delays.md +++ b/_includes/doc/admin-guide/options/log-fetch-delays.md @@ -8,7 +8,7 @@ **NOTE:** Increasing the value of this parameter (which reduces the delay time) can improve log feed performance, but it may also increase system load. {: .notice--info} -{% include doc/admin-guide/options/deprecated-options.md old='fetch-delay' new='log-fetch-delay' %} +{% include doc/admin-guide/options/deprecated-options.md old='fetch-delay()' new='log-fetch-delay()' %} ## log-fetch-retry-delay() @@ -17,4 +17,4 @@ *Description:* Controls how many seconds {{ site.product.short_name }} remains idle before checking for new logs, in case no new logs were read during the previous check. -{% include doc/admin-guide/options/deprecated-options.md old='fetch-retry-delay' new='log-fetch-retry-delay' %} +{% include doc/admin-guide/options/deprecated-options.md old='fetch-retry-delay()' new='log-fetch-retry-delay()' %} diff --git a/_includes/doc/admin-guide/options/log-prefix.md b/_includes/doc/admin-guide/options/log-prefix.md index ca0613eb..c06f64cf 100644 --- a/_includes/doc/admin-guide/options/log-prefix.md +++ b/_includes/doc/admin-guide/options/log-prefix.md @@ -1,11 +1,3 @@ -## log-prefix() (DEPRECATED) +## log-prefix() -| Type: | string| -|Default:|| - -*Description:* A string added to the beginning of every log message. It -can be used to add an arbitrary string to any log source, though it is -most commonly used for adding kernel: to the kernel messages on Linux. - -**NOTE:** This option is deprecated. Use **program-override** instead. -{: .notice--info} \ No newline at end of file +{% include doc/admin-guide/options/deprecated-options.md old='log-prefix()' new='program-override()' %} \ No newline at end of file diff --git a/doc/_admin-guide/070_Destinations/130_MongoDB/README.md b/doc/_admin-guide/070_Destinations/130_MongoDB/README.md index 5276e41f..6b53a251 100644 --- a/doc/_admin-guide/070_Destinations/130_MongoDB/README.md +++ b/doc/_admin-guide/070_Destinations/130_MongoDB/README.md @@ -62,5 +62,3 @@ destination d_mongodb { ); }; ``` - -{% include doc/admin-guide/options/deprecated-options.md old='libmongo client syntax' new='mongodb' %} diff --git a/doc/_admin-guide/090_Global_options/000_Global_options.md b/doc/_admin-guide/090_Global_options/000_Global_options.md index 019019e5..21384f25 100644 --- a/doc/_admin-guide/090_Global_options/000_Global_options.md +++ b/doc/_admin-guide/090_Global_options/000_Global_options.md @@ -274,7 +274,7 @@ Higher log-levels automatically include messages from lower log-levels: {% include doc/admin-guide/options/log-msg-size.md %} -## mark() (DEPRECATED) +## mark() {% include doc/admin-guide/options/deprecated-options.md old='mark()' new='mark-freq()' %} @@ -299,7 +299,7 @@ Starting with version 3.16, the default value of this option is -1, so {{ site.product.short_name }} does not change the ownership, unless explicitly configured to do so. -## pass-unix-credentials() (DEPRECATED) +## pass-unix-credentials() {% include doc/admin-guide/options/deprecated-options.md old='pass-unix-credentials()' new='so-passcred()' %} @@ -448,19 +448,19 @@ Possible values: - auto: Use the settings of the stats-level() option. -## stats-freq() (DEPRECATED) +## stats-freq() {% include doc/admin-guide/options/deprecated-options.md old='stats-freq()' new='stats(freq())' %} -## stats-level() (DEPRECATED) +## stats-level() {% include doc/admin-guide/options/deprecated-options.md old='stats-level()' new='stats(level())' %} -## stats-max-dynamics() (DEPRECATED) +## stats-max-dynamics() {% include doc/admin-guide/options/deprecated-options.md old='stats-max-dynamics()' new='stats(max-dynamics())' %} -## sync() or sync-freq() (DEPRECATED) +## sync() or sync-freq() {% include doc/admin-guide/options/deprecated-options.md old='sync() and sync-freq()' new='flush-lines()' %} diff --git a/doc/_dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver.md b/doc/_dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver.md index f64db628..7cd7f1aa 100644 --- a/doc/_dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver.md +++ b/doc/_dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver.md @@ -1,5 +1,5 @@ --- -title: file() Source Driver (DEPRECATED) +title: file() Source Driver description: >- The file() source has been deprecated. For a more stable performance, use the [[wildcard-file() source|adm-src-wild]] instead. The file() source driver is used to collect log messages from plain-text files, for example, from the logfiles of an Apache webserver. From 6296ac3b31123d81556b11dcf6368ed65fa6ae3d Mon Sep 17 00:00:00 2001 From: Hofi Date: Tue, 25 Nov 2025 17:39:17 +0100 Subject: [PATCH 3/6] css: adjust heading font sizes Signed-off-by: Hofi --- .../minimal-mistakes/minimal-mistakes/_variables.scss | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_sass/minimal-mistakes/minimal-mistakes/_variables.scss b/_sass/minimal-mistakes/minimal-mistakes/_variables.scss index f0cd9bf0..3b407523 100644 --- a/_sass/minimal-mistakes/minimal-mistakes/_variables.scss +++ b/_sass/minimal-mistakes/minimal-mistakes/_variables.scss @@ -45,11 +45,11 @@ $type-size-8: 0.625em !default; // ~10px $type-size-9: 0.5em !default; // ~8px /* headline scale */ -$h-size-1: 1.563em !default; // ~25.008px -$h-size-2: 1.25em !default; // ~20px -$h-size-3: 1.125em !default; // ~18px -$h-size-4: 1.0625em !default; // ~17px -$h-size-5: 1.03125em !default; // ~16.5px +$h-size-1: 1.6em !default; // ~25.68px +$h-size-2: 1.4em !default; // ~22.4px +$h-size-3: 1.2em !default; // ~19.2px +$h-size-4: 1.1em !default; // ~17.6px +$h-size-5: 1.05em !default; // ~16.8px $h-size-6: 1em !default; // ~16px /* From a430c8a9f1013d8ba2e98e1a1222dd48bd4715d7 Mon Sep 17 00:00:00 2001 From: Hofi Date: Tue, 25 Nov 2025 17:40:04 +0100 Subject: [PATCH 4/6] darwin-osl: update changed, add new options Signed-off-by: Hofi --- .../options/ignore-saved-bookmarks.md | 6 +++++ .../admin-guide/options/log-fetch-delays.md | 8 +++--- .../admin-guide/options/read-old-records.md | 6 +++++ .../085_macOS/000_darwin_oslog_options.md | 26 ++++++++----------- .../01_collecting-native-logs.md | 2 +- 5 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 _includes/doc/admin-guide/options/ignore-saved-bookmarks.md create mode 100644 _includes/doc/admin-guide/options/read-old-records.md diff --git a/_includes/doc/admin-guide/options/ignore-saved-bookmarks.md b/_includes/doc/admin-guide/options/ignore-saved-bookmarks.md new file mode 100644 index 00000000..eb1bab77 --- /dev/null +++ b/_includes/doc/admin-guide/options/ignore-saved-bookmarks.md @@ -0,0 +1,6 @@ +## ignore-saved-bookmarks() + +|Type:| boolean| +|Default:| no| + +*Description:* By default, {{ site.product.short_name }} continues reading logs from the last remembered position (or offset, etc.) stored in its persist file after a restart or reload. If this option is set to `yes`, it will always start reading from either the beginning or the end of the available log list \ No newline at end of file diff --git a/_includes/doc/admin-guide/options/log-fetch-delays.md b/_includes/doc/admin-guide/options/log-fetch-delays.md index f9f2f11f..fd710dd5 100644 --- a/_includes/doc/admin-guide/options/log-fetch-delays.md +++ b/_includes/doc/admin-guide/options/log-fetch-delays.md @@ -1,7 +1,7 @@ ## log-fetch-delay() -|Type:| integer| -|Default:| `10000`| +| Type: | integer [1 second / fetch_retry_delay * 1000000 milliseconds] | +|Default:| 10000 (10 milliseconds)| *Description:* Sets the time {{ site.product.short_name }} waits between reading and sending log messages. The unit of this parameter is a fraction of a second, where `wait_time = 1 second / `. For example, setting it to `1` results in approximately one log being read/sent per second, while `1000000` means a delay of only 1 microsecond between read/write attempts. The maximum value of this parameter is `1000000`. @@ -12,8 +12,8 @@ ## log-fetch-retry-delay() -|Type:| integer| -|Default:| `1`| +|Type:| integer in seconds | +|Default:| 1| *Description:* Controls how many seconds {{ site.product.short_name }} remains idle before checking for new logs, in case no new logs were read during the previous check. diff --git a/_includes/doc/admin-guide/options/read-old-records.md b/_includes/doc/admin-guide/options/read-old-records.md new file mode 100644 index 00000000..e704e9b5 --- /dev/null +++ b/_includes/doc/admin-guide/options/read-old-records.md @@ -0,0 +1,6 @@ +## read-old-records() + +|Type:| boolean| +|Default:| no| + +*Description:* If read-old-record() is set to `yes`, {{ site.product.short_name }} will start fetching from the oldest available message; otherwise, it will start from the newest one (if no bookmarks are present, or ignore-saved-bookmarks() is set to `yes`). diff --git a/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md b/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md index 5198c2cc..fb53cf65 100644 --- a/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md +++ b/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md @@ -2,7 +2,7 @@ title: darwin-oslog() source options id: adm-src-darw-osl-opt description: >- - The `darwin-oslog()` source is based on the native OSLog Framework to read logs from the local store of the unified logging system on Darwin OSes. The {{ site.product.short_name }} `system()` source automatically uses this new source on Darwin platforms if the `darwinosl` plugin is available. This plugin is available only on macOS 10.15 Catalina and later versions. The 10.15 version is the first to support the OSLog API. + The darwin-oslog() source is based on the native OSLog Framework to read logs from the local store of the unified logging system on Darwin OSes. The {{ site.product.short_name }} system() source automatically uses this new source on Darwin platforms if the `darwinosl` plugin is available. This plugin is available only on macOS 10.15 Catalina and later versions. The 10.15 version is the first to support the OSLog API. --- **NOTE:** The persistent OSLog store keeps about 7 days of logs on the disk. @@ -17,20 +17,21 @@ The `darwin-oslog()` source has the following options. *Description:* String for native macOS log message filtering using predicates. For example, the following predicate selects AirDrop logs: `subsystem=="com.apple.sharing" and category=="AirDrop"` -## do-not-use-bookmark() - -|Type:| boolean| -|Default:| `no`| - -*Description:* By default, {{ site.product.short_name }} continues to read the logs from the last remembered position after a restart. If this option is set to `yes`, it always starts reading from the end or beginning of the available log list (depending on the setting of the `go-reverse()` and the `read-old-records()` options. +{% include doc/admin-guide/options/disable-bookmarks.md %} ## go-reverse() |Type:| boolean| -|Default:| `no`| +|Default:| no| *Description:* If set to `yes`, the logs are processed in reverse order (latest to oldest). +{% include doc/admin-guide/options/hook.md %} + +{% include doc/admin-guide/options/ignore-saved-bookmarks.md %} (depending on the setting of the go-reverse() and the read-old-records() options. + +{% include doc/admin-guide/options/deprecated-options.md old='do-not-use-bookmark()' new='ignore-saved-bookmarks()' %} + {% include doc/admin-guide/options/log-fetch-delays.md %} {% include doc/admin-guide/options/log-fetch-limit.md %} @@ -38,13 +39,8 @@ The `darwin-oslog()` source has the following options. ## max-bookmark-distance() |Type:| integer| -|Default:| `0`(unlimited) [seconds]| +|Default:| 0 (unlimited) in seconds| *Description:* The maximum distance in seconds that a bookmark can point backward. That is, if {{ site.product.short_name }} is stopped for 10 minutes (600 seconds) and `max-bookmark-distance()` is set to `60`, then {{ site.product.short_name }} starts reading the logs from 60 seconds before the startup, missing 9 minutes (540 seconds) worth of logs. -## read-old-records() - -|Type:| boolean| -|Default:| `no`| - -*Description:* If set to yes, {{ site.product.short_name }} starts reading logs from the oldest available entry when the system starts, or if no bookmarks are present. +{% include doc/admin-guide/options/read-old-records.md %} diff --git a/doc/_dev-guide/chapter_4/section_2/macos-testing-status/system-source/01_collecting-native-logs.md b/doc/_dev-guide/chapter_4/section_2/macos-testing-status/system-source/01_collecting-native-logs.md index 6d509cba..b107fbf0 100644 --- a/doc/_dev-guide/chapter_4/section_2/macos-testing-status/system-source/01_collecting-native-logs.md +++ b/doc/_dev-guide/chapter_4/section_2/macos-testing-status/system-source/01_collecting-native-logs.md @@ -35,7 +35,7 @@ For more info, see [oslog](https://developer.apple.com/documentation/oslog?langu - boolean value, setting to `yes` will provide a reverse-ordered log list (from latest to oldest) - default value: `no` -- `do-not-use-bookmark()` +- `ignore-saved-bookmarks()` - boolean value, setting to `yes` will prevent {{ site.product.short_name }} from continuing to feed the logs from the last remembered position after a (re-)start, which means, depending on the other settings, the feed will always start from the end/beginning From b16edc7da46cc83d1fb23a76301b119c5110a68c Mon Sep 17 00:00:00 2001 From: Hofi Date: Wed, 26 Nov 2025 15:04:40 +0100 Subject: [PATCH 5/6] kafka: added kafka source, updated kafka dest Signed-off-by: Hofi --- _data/link_aliases.yml | 6 + _data/navigation.yml | 7 +- .../admin-guide/options/bootstrap-servers.md | 17 ++ .../doc/admin-guide/options/config-kafka.md | 20 +++ .../admin-guide/options/disable-bookmarks.md | 9 ++ .../doc/admin-guide/options/kafka-logging.md | 13 ++ .../options/kafka-source-workers.md | 6 + .../038_Kafka/001_Kafka_options.md | 148 ++++++++++++++++++ .../060_Sources/038_Kafka/README.md | 71 +++++++++ .../100_Kafka-c/003_Kafka-c_options.md | 110 ++++--------- .../070_Destinations/100_Kafka-c/README.md | 3 +- 11 files changed, 331 insertions(+), 79 deletions(-) create mode 100644 _includes/doc/admin-guide/options/bootstrap-servers.md create mode 100644 _includes/doc/admin-guide/options/config-kafka.md create mode 100644 _includes/doc/admin-guide/options/disable-bookmarks.md create mode 100644 _includes/doc/admin-guide/options/kafka-logging.md create mode 100644 _includes/doc/admin-guide/options/kafka-source-workers.md create mode 100644 doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md create mode 100644 doc/_admin-guide/060_Sources/038_Kafka/README.md diff --git a/_data/link_aliases.yml b/_data/link_aliases.yml index 6177bf82..cd76e211 100644 --- a/_data/link_aliases.yml +++ b/_data/link_aliases.yml @@ -117,6 +117,12 @@ adm-src-program: adm-dest-program: aliases: [ "program() destination" ] +adm-src-kafka: + aliases: [ "kafka() source" ] + +adm-dest-kafkac: + aliases: [ "kafka() destination" ] + adm-src-mqtt: aliases: [ "mqtt() source" ] diff --git a/_data/navigation.yml b/_data/navigation.yml index 21fb824a..9384fdcf 100644 --- a/_data/navigation.yml +++ b/_data/navigation.yml @@ -165,6 +165,11 @@ admin-guide-nav: - title: "Jellyfin" url: /admin-guide/060_Sources/035_Jellyfin/README subnav: + - title: "kafka" + url: /admin-guide/060_Sources/038_Kafka/README + subnav: + - title: "Options of the kafka() source" + url: /admin-guide/060_Sources/038_Kafka/001_Kafka_options - title: "kubernetes" url: /admin-guide/060_Sources/040_Kubernetes/README subnav: @@ -1159,7 +1164,7 @@ dev-guide-nav: subnav: - title: "file() Destination Driver" url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/file-destination-driver - - title: "file() Source Driver (DEPRECATED)" + - title: "file() Source Driver" url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver - title: "pipe() Destination Driver" url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/pipe-destination-driver diff --git a/_includes/doc/admin-guide/options/bootstrap-servers.md b/_includes/doc/admin-guide/options/bootstrap-servers.md new file mode 100644 index 00000000..2f869116 --- /dev/null +++ b/_includes/doc/admin-guide/options/bootstrap-servers.md @@ -0,0 +1,17 @@ +## bootstrap-servers() + +| Type: | string | +|Default: | N/A | +|Mandatory:| yes | + +*Description:* Specifies the hostname or IP address of the Kafka server. +When specifying an IP address, IPv4 (for example, 192.168.0.1) or IPv6 +(for example, \[::1\]) can be used as well. Use a colon (**:**) after +the address to specify the port number of the server. When specifying +multiple addresses, use a comma to separate the addresses, for example: + +``` config +bootstrap-servers( + "127.0.0.1:2525,remote-server-hostname:6464" +) +``` diff --git a/_includes/doc/admin-guide/options/config-kafka.md b/_includes/doc/admin-guide/options/config-kafka.md new file mode 100644 index 00000000..2513f5b4 --- /dev/null +++ b/_includes/doc/admin-guide/options/config-kafka.md @@ -0,0 +1,20 @@ +## config() + +| Type: | key-value pairs | +|Default:| N/A | + +*Description:* You can use this option to set the properties of the kafka {{ include.kafka_type }}. + +The {{ site.product.short_name }} kafka {{ include.type }} supports all properties of the official Kafka {{ include.kafka_type }}. For details, see the librdkafka documentation. + +The syntax of the config() option is the following: + +``` config +config( + “key1” => “value1” + “key2” => “value2” +) +``` + +**NOTE:** The following kafka {{ include.kafka_type }} config options are protected and cannot be overriden in the `config()` list: {{ include.protected_options }} +{: .notice--info} diff --git a/_includes/doc/admin-guide/options/disable-bookmarks.md b/_includes/doc/admin-guide/options/disable-bookmarks.md new file mode 100644 index 00000000..d1997706 --- /dev/null +++ b/_includes/doc/admin-guide/options/disable-bookmarks.md @@ -0,0 +1,9 @@ +## disable-bookmarks() + +| Type: | boolean | +| Default: | no | + +*Description:* This option prevents {{ site.product.short_name }} from storing a bookmark (such as position or offset) in its persist file for the last processed message. + +**NOTE:** This will not prevent usage of an already presented bookmark entry, for ignoring those bookmark entries specify `ignore-saved-bookmarks(yes)` as well. +{: .notice--info} \ No newline at end of file diff --git a/_includes/doc/admin-guide/options/kafka-logging.md b/_includes/doc/admin-guide/options/kafka-logging.md new file mode 100644 index 00000000..c733ceb7 --- /dev/null +++ b/_includes/doc/admin-guide/options/kafka-logging.md @@ -0,0 +1,13 @@ +## kafka-logging() + +| Accepted values: | disabled \| trace \| kafka | +| Default: | disabled | + +*Description:* This option allows you to control how internal Kafka logs appear in the {{ site.product.short_name }} logs. + +- disabled: Disables internal Kafka log messages in the {{ site.product.short_name }} logs. +- trace: Logs all internal Kafka messages at the `trace` level of {{ site.product.short_name }}. +- kafka: Logs internal Kafka messages using log levels mapped to those of {{ site.product.short_name }}. + +**NOTE:** The internal Kafka logging level itself can be configured using the config() Kafka options. For details, refer to the librdkafka documentation. +{: .notice--info} diff --git a/_includes/doc/admin-guide/options/kafka-source-workers.md b/_includes/doc/admin-guide/options/kafka-source-workers.md new file mode 100644 index 00000000..aec3d0d9 --- /dev/null +++ b/_includes/doc/admin-guide/options/kafka-source-workers.md @@ -0,0 +1,6 @@ +- One **main worker** that fetches messages from the Kafka broker and stores them into an internal queue. +- A second worker that processes the queued messages and forwards them to the configured destination. + +Although the source can operate using a single worker, this configuration typically results in a significant performance penalty compared to the default multi-worker setup. + +Increasing the number of workers beyond two may further improve throughput, especially when the main worker can fetch messages at high speed. In such cases, you may also need to fine-tune related options such as separated-worker-queues(), log-fetch-limit(), log-fetch-delay(), log-fetch-retry-delay(), log-fetch-queue-full-delay(). diff --git a/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md b/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md new file mode 100644 index 00000000..85a1ca7e --- /dev/null +++ b/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md @@ -0,0 +1,148 @@ +--- +title: "Options of the kafka() source" +id: adm-src-kafka-opt +description: >- + This section describes the options of the kafka() source in {{ site.product.short_name }}. +--- + +The kafka() source of {{ site.product.short_name }} can directly consume log messages from the Apache Kafka message bus. The source has the following options. + +## Required options + +To use the kafka() source, the following two options are required: bootstrap-servers() and topic(). Both must appear at the beginning of your {{ site.product.short_name }} configuration. + +{% include doc/admin-guide/options/bootstrap-servers.md %} + +{% include doc/admin-guide/options/config-kafka.md kafka_type='consumer' type='source' protected_options='`bootstrap.servers` `metadata.broker.list` `enable.auto.offset.store` `auto.offset.reset` `enable.auto.commit` `auto.commit.enable`' %} + +{% include doc/admin-guide/options/disable-bookmarks.md %} +See Bookmarking in the kafka() source for more details. + +{% include doc/admin-guide/options/hook.md %} + +{% include doc/admin-guide/options/ignore-saved-bookmarks.md %} (depending on the setting of the read-old-records() option.\ +See Bookmarking in the kafka() source for more details. + +{% include doc/admin-guide/options/kafka-logging.md %} + +## log-fetch-limit() + +| Type: | integer | +|Default:| 10000 | + +*Description:* Specifies the maximum number of messages the main worker will consume and queue from the Kafka broker. This effectively determines the size of the internally used Kafka message queue. If the limit is reached, the kafka() source stops fetching messages from the broker, logs the situation, and waits the amount of time specified by fetch-queue-full-delay() before attempting to fetch new data again. + +**NOTE:** If more than 2 workers are configured and separated-worker-queues() is set to `yes`, then all processor workers share this total queue size. +For example, with `workers(3)` and `fetch-limit(100000)`, the 2 processor workers (remember, the first of the configured 3 is always the main worker) will each receive their own queue, and neither queue will grow beyond 50,000 messages. +{: .notice--info} + +**NOTE:** This options worth align with the kafka config options `queued.min.messages` and `queued.max.messages.kbytes`, For details, refer to the librdkafka documentation. +{: .notice--info} + +## log-fetch-delay() + +| Type: | integer [1 second / fetch_retry_delay * 1000000 milliseconds] | +|Default:| 1000 (1 millisecond) | + +*Description:* Specifies the time the main worker will wait between attempts to fetch new data. + +## log-fetch-retry-delay() + +| Type: | integer [1 second / fetch_retry_delay * 1000000 milliseconds] | +|Default:| 10000 (10 milliseconds)| + +*Description:* Specifies the time the main worker will wait before attempting to fetch new data again when the broker signals no more data is available. + +## log-fetch-queue-full-delay() + +| Type: | integer in milliseconds | +|Default:| 1000 | + +*Description:* When the main worker reaches the queued message limit defined by fetch-limit(), the kafka() source temporarily stops retrieving messages from the broker. It then waits for the duration specified by `fetch-queue-full-delay()` before attempting to fetch additional messages. + +{% include doc/admin-guide/options/persist-name.md %} + +## poll-timeout() + +| Type: | integer in milliseconds | +|Default:| 10000 | + +*Description:* Specifies the maximum amount of time {{ site.product.short_name }} waits during a Kafka broker poll request for new messages to become available. + +{% include doc/admin-guide/options/read-old-records.md %}\ +See Bookmarking in the kafka() source for more details. + +## separated-worker-queues() + +| Type: | yes \| no | +|Default:| no | + +*Description:* When the value of workers() is greater than 2 (meaning multiple processor threads are used to handle queued messages), and `separated-worker-queues()` is set to `yes`, the main worker of the kafka() source distributes the consumed messages into separate queues, one for each processor worker. + +**NOTE:** This approach can improve performance, especially in high-throughput scenarios, but may also lead to significantly increased memory usage. +{: .notice--info} + +## strategy-hint() + +| Accepted values: | assign, subscribe | +| Default: | assign | + +*Description:* This option provides a hint about which Kafka consumer strategy the kafka() source should use when the topic() list contains topic/partition definitions that could be handled in either way. + +Why is it worth using dual consumer strategies? describes the differences between the two. + +For details about how the resulting topic names, partitions, and Kafka assign/subscribe strategies are determined in different scenarios, see Basic startegy usage cross-reference of the different topic configuration cases + +## time-reopen() + +| Type: | integer in seconds | +|Default:| 60 | + +*Description:* The time {{ site.product.short_name }} waits between attempts to recover from errors that require re-initialization of the full kafka connection and its internally used data structures. + +## topic() + +| Type: | key-value pairs | +|Default: | N/A | +|Mandatory:| yes | + +*Description:* A list of pairs consisting of Kafka topic name(s) and partition number(s) from which messages are consumed, for example: + +``` config +topic( + "^topic-name-[13]$" => "-1" + "topic-name-2" => "1" + "topic-name-4" => "-1" + "topic-name-5" => "0,1,4" +} +``` + +Valid topic names have the following limitations: + +- The topic name must either contain only characters matching the pattern `[-._a-zA-Z0-9]`, or it can be a regular expression. + For example: `^topic-name-[13]$` (which expands to `topic-name-1` and `topic-name-3`). +- The length of the topic name must be between 1 and 249 characters. + +The partition number must be: + +- either a single partition number or a comma-separated list of partition numbers +- a positive integer, or `-1`, which means all partitions of the topic + +For details about how the resulting topic names, partitions, and Kafka assign/subscribe strategies are determined in different scenarios, see Basic startegy usage cross-reference of the different topic configuration cases and Why is it worth using dual consumer strategies? + +## workers() + +| Type: | integer | +|Default:| 2 | + +*Description:* The number of workers the `kafka()` source uses to consume and process messages from the kafka broker. By default, uses two of them: + +{% include doc/admin-guide/options/kafka-source-workers.md %} + +![]({{ site.baseurl}}/assets/images/caution.png) **CAUTION:** +Only kafka() sources with `workers()` set to less than 3 can guarantee ordered message forwarding. +{: .notice--warning} + +**NOTE:** Kafka clients have their own threadpool, entirely independent from +any {{ site.product.short_name }} settings. The `workers()` option has no effect on this threadpool. +{: .notice--info} diff --git a/doc/_admin-guide/060_Sources/038_Kafka/README.md b/doc/_admin-guide/060_Sources/038_Kafka/README.md new file mode 100644 index 00000000..6991360d --- /dev/null +++ b/doc/_admin-guide/060_Sources/038_Kafka/README.md @@ -0,0 +1,71 @@ +--- +title: 'kafka(): Consuming messages from Apache Kafka using the librdkafka client' +short_title: kafka +id: adm-src-kafka +description: >- + Starting with version 4.11, {{ site.product.name }} can directly fetch log messages from the Apache Kafka message bus. +--- + +The kafka() source can fetch messages from explicitly named or wildcard-matching Kafka topics, and from a single partition, +explicitly listed partitions, or all partitions of the selected topic(s). It can use two different strategies +— `assign` or `subscribe` — to start consuming messages from the selected partition(s). +The strategy is determined automatically based on the topic() option definitions and the strategy-hint() option.\ +The basic rule is the following: + +`subscribe` is used if the topic name contains characters that are not allowed in standard Kafka topic names +(in which case the topic name is treated as a regular expression), if the partition number is `-1`, or if the value +of strategy-hint() is `subscribe` (except when multiple partition numbers are provided for the +same topic name — this will raise an error). + +`assign` (the default) is used if the topic name contains only valid Kafka topic characters (for example, +no regexp-related characters) and only positive partition numbers are specified. + +## Basic startegy usage cross-reference of the different topic configuration cases + +| topic(...) in config | topic name(s) | part. number(s) | strategy-hint() | resulting strategy | +|-------------------------------------------------------|----------------------------|-----------------|-----------------|--------------------| +| topic( "topic-name-1" => "1" } | topic-name-1 | 1 | assign | assign | +| topic( "topic-name-1" => "1" } | topic-name-1 | 1 | subscribe | subscribe | +| topic( "topic-name-1" => "1,2" } | topic-name-1 | 1-2 | assign | assign | +| topic( "topic-name-1" => "1,2" } | topic-name-1 | 1-2 | subscribe | N/A (error) | +| topic( "topic-name-1" => "1" "topic-name-1" => "2" } | topic-name-1 | 1-2 | assign | assign | +| topic( "topic-name-1" => "1" "topic-name-1" => "2" } | topic-name-1 | 1-2 | subscribe | N/A (error) | +| topic( "topic-name-1" => "1" "topic-name-3" => "2" } | topic-name-1, topic-name-3 | 1, 2 | assign | assign | +| topic( "topic-name-1" => "1" "topic-name-3" => "2" } | topic-name-1, topic-name-3 | 1, 2 | subscribe | subscribe | +| topic( "topic-name-1" => "-1" } | topic-name-1 | all | assign | subscribe | +| topic( "topic-name-1" => "-1" } | topic-name-1 | all | subscribe | subscribe | +| topic( "topic-name-1" => "1" "topic-name-3" => "-1" } | topic-name-1, topic-name-3 | 1, all | assign | subscribe | +| topic( "topic-name-1" => "1" "topic-name-3" => "-1" } | topic-name-1, topic-name-3 | 1, all | subscribe | subscribe | +| topic( "topic-name-3" => "1" "topic-name-3" => "-1" } | topic-name-1, topic-name-3 | 1, all | assign | subscribe | +| topic( "topic-name-3" => "1" "topic-name-3" => "-1" } | topic-name-1, topic-name-3 | 1, all | subscribe | subscribe | +| topic( "^topic-name-[13]$" => "2" } | topic-name-1, topic-name-3 | 2, 2 | assign | subscribe | +| topic( "^topic-name-[13]$" => "2" } | topic-name-1, topic-name-3 | 2, 2 | subscribe | subscribe | +| topic( "^topic-name-[13]$" => "-1" } | topic-name-1, topic-name-3 | all, all | assign | subscribe | +| topic( "^topic-name-[13]$" => "-1" } | topic-name-1, topic-name-3 | all, all | subscribe | subscribe | + +## Why is it worth using dual consumer strategies? + +Using both consumer strategies — `assign` and `subscribe` — provides the flexibility to adapt to a wide range of Kafka setups and practical use cases, instead of forcing a single approach that may not fit all scenarios. + +- `assign` is ideal when full control and predictability are required. + - You can explicitly target a known set of topics and partitions. + - Guarantees ordering semantics more reliably in single-partition or controlled multi-partition scenarios. + - Works well in environments where the topic layout is static and predefined. + +- `subscribe` is valuable when flexibility matters more than strict control. + - It supports regular expressions, making it suitable when topic names follow patterns or when topics may appear dynamically. + - It automatically handles partition assignments inside a consumer group, reducing configuration overhead. + - It integrates better with scaling scenarios or when consumers should share workload automatically. + - The possible drawbacks of unordered and/or repeated messages are acceptable. + +By supporting both approaches, {{ site.product.short_name }} can be used effectively in a variety of Kafka consumption models — from tightly controlled, partition-specific pipelines to dynamic and scalable consumer setups that evolve with the broker configuration. + +## Bookmarking in the kafka() source + +By default, {{ site.product.short_name }} stores the offset of the last read message of each topic it consumes in its own persist file. This can be disabled using the disable-bookmarks() option. Automatic offset restoration takes effect at startup or reload, based on the saved offset value and the ignore-saved-bookmarks() and read-old-record() settings. If ignore-saved-bookmarks() is set to `yes`, it will not use the saved offset. Instead, if read-old-record() is set to `yes`, it will start fetching from the oldest available message, otherwise it will start from the newest one. + +## Multiple workers in the kafka() source + +The kafka() source can fetch and process messages from the fafka broker using multiple workers(), by default 2 of them: + +{% include doc/admin-guide/options/kafka-source-workers.md %} diff --git a/doc/_admin-guide/070_Destinations/100_Kafka-c/003_Kafka-c_options.md b/doc/_admin-guide/070_Destinations/100_Kafka-c/003_Kafka-c_options.md index cc598669..bf92f310 100644 --- a/doc/_admin-guide/070_Destinations/100_Kafka-c/003_Kafka-c_options.md +++ b/doc/_admin-guide/070_Destinations/100_Kafka-c/003_Kafka-c_options.md @@ -8,18 +8,12 @@ description: >- The kafka() destination of {{ site.product.short_name }} can directly publish log messages to the Apache Kafka message bus, where subscribers can access them. The destination has the following options. -**NOTE:** `kafka()` and `kafka-c()` is now interchangable as the first one is just an alias of the latter +**NOTE:** `kafka()` and `kafka-c()` is now interchangable as the latter is just an alias of the first one. {: .notice--info} ## Required options -To use the kafka() destination, the following two options are required: `bootstrap-servers()` and `topic()`. Both must appear at the beginning of your {{ site.product.short_name }} configuration. - -You can specify multiple, comma-separated addresses, demonstrated in the following example: - -```config -bootstrap-servers("127.0.0.1:2525,remote-server-hostname:6464") -``` +To use the kafka() destination, the following two options are required: bootstrap-servers() and topic(). Both must appear at the beginning of your {{ site.product.short_name }} configuration. {% include doc/admin-guide/options/batch-lines.md %} @@ -51,46 +45,9 @@ For more information about the default values of the transaction.timeout.ms Kafka property, see the librdkafka documentation. -## bootstrap-servers() - -| Type:| string| - |Default:| | - -*Description:* Specifies the hostname or IP address of the Kafka server. -When specifying an IP address, IPv4 (for example, 192.168.0.1) or IPv6 -(for example, \[::1\]) can be used as well. Use a colon (**:**) after -the address to specify the port number of the server. When specifying -multiple addresses, use a comma to separate the addresses, for example, -bootstrap-servers(\"127.0.0.1:2525,remote-server-hostname:6464\") - -For the kafka destination, include the path to the directory where you -copied the required libraries (see [[Prerequisites|adm-dest-hdfs-pre]]), for example, -**client-lib-dir("/opt/syslog-ng/lib/syslog-ng/java-modules/KafkaDestination.jar:/usr/share/kafka/lib/*.jar")**. - -**NOTE:** Unlike in the Java implementation, the client-lib-dir() option has -no significant role in the C implementation of the kafka() destination. -The programming language accepts this option for better compatibility. -{: .notice--info} - - -## config() +{% include doc/admin-guide/options/bootstrap-servers.md %} -| Type:| | - |Default:| | - -*Description:* You can use this option to set the properties of the kafka producer. - -The {{ site.product.short_name }} kafka destination supports all properties of the -official Kafka producer. For details, see the librdkafka documentation. - -The syntax of the config() option is the following: - -```config -config( - “key1” => “value1” - “key2” => “value2” -) -``` +{% include doc/admin-guide/options/config-kafka.md kafka_type='producer' type='destination' protected_options='`bootstrap.servers` `metadata.broker.list`' %} {% include doc/admin-guide/options/disk-buffer.md %} @@ -99,16 +56,16 @@ config( | Type:| string| |Default:| N/A| -*Description:* If the resolved `topic()` template is not a valid Kafka topic , {{ site.product.short_name }} will use `fallback-topic()` to send messages. +*Description:* If the resolved topic() template is not a valid Kafka topic , {{ site.product.short_name }} will use `fallback-topic()` to send messages. -**NOTE:** If instead of strings, you use actual templates (that is, a macro like `${MESSAGE}`, or a template function like `$(format-json)`) in the `topic()` option, configuring the `fallback-topic()` option is required. +**NOTE:** If instead of strings, you use actual templates (that is, a macro like ${MESSAGE}, or a template function like $(format-json)) in the topic() option, configuring the `fallback-topic()` option is required. {: .notice--info} {% include doc/admin-guide/options/frac-digits.md %} ## flush-timeout-on-reload() -| Type:| integer in msec| +| Type:| integer in milliseconds| |Default:| 1000| *Description:* When {{ site.product.short_name }} reloads, the Kafka client will also @@ -120,7 +77,7 @@ without disk-buffering, too. ## flush-timeout-on-shutdown() -| Type:| integer in msec| +| Type:| integer in milliseconds| |Default:| 60000| *Description:* When {{ site.product.short_name }} shuts down, the Kafka client will also @@ -151,12 +108,11 @@ key(\"${PROGRAM}\"). |Default:| ${ISODATE} ${HOST} ${MSGHDR}${MSG}\\n| *Description:* The message as published to Apache Kafka. You can use -templates and template functions (for example, format-json()) to format -the message, for example, template(\"$(format-json \--scope rfc5424 -\--exclude DATE \--key ISODATE)\"). +templates and template functions to format +the message, for example, `template(\"$(format-json \--scope rfc5424 +\--exclude DATE \--key ISODATE)\")`. -For details on formatting messages in JSON format, see -[[format-json|adm-temp-func#format-json]]. +For details on formatting messages in JSON format, see $format-json() {% include doc/admin-guide/options/on-error.md %} @@ -164,13 +120,13 @@ For details on formatting messages in JSON format, see ## poll-timeout() -| Type:| integer in msec| - |Default:| 1000| +| Type: | integer in milliseconds | +|Default:| 10000 | *Description:* Specifies the frequency your {{ site.product.short_name }} queries the Kafka -client about the amount of messages sent since the last poll-timeout (). +client about the amount of messages sent since the last `poll-timeout()`. In case of multithreading, the first {{ site.product.short_name }} worker is responsible for -poll-timeout(). +`poll-timeout()`. {% include doc/admin-guide/options/retries.md %} @@ -178,10 +134,10 @@ poll-timeout(). ## sync-send() -| Type:| yes \| no| - |Default:| no| +| Type: | yes \| no| +|Default:| no| -*Description:* When sync-send is set to **yes**, {{ site.product.short_name }} sends +*Description:* When `sync-send()` is set to `yes`, {{ site.product.short_name }} sends the message reliably: it sends a message to the Kafka server, then waits for a reply. In case of failure, {{ site.product.short_name }} repeats sending the message, as set in the retries() parameter. If sending the message fails @@ -189,7 +145,7 @@ for retries() times, {{ site.product.short_name }} drops the message. This method ensures reliable message transfer, but is very slow. -When sync-send() is set to **no**, {{ site.product.short_name }} sends messages +When `sync-send()` is set to `no`, {{ site.product.short_name }} sends messages asynchronously, and receives the response asynchronously. In case of a problem, {{ site.product.short_name }} cannot resend the messages. @@ -197,9 +153,9 @@ This method is fast, but the transfer is not reliable. Several thousands of messages can be lost before {{ site.product.short_name }} recognizes the error. ![]({{ site.baseurl}}/assets/images/caution.png) **CAUTION:** -Hazard of data loss! If sync-send() is set to "no", the messages passed +Hazard of data loss! If `sync-send()` is set to "no", the messages passed to the Kafka client can be lost. To avoid data loss, One Identity -recommends that you set sync-send() to "yes", as this setting +recommends that you set `sync-send()` to "yes", as this setting delivers messages to the Kafka client more reliably. {: .notice--danger} @@ -209,27 +165,29 @@ delivers messages to the Kafka client more reliably. ## time-reopen() -| Type:| number (seconds)| - |Default:| 60| +| Type: | integer in seconds | +|Default:| 60| -*Description:* This is an optional parameter. If message delivery fails, {{ site.product.short_name }} retries sending the messages for `retries()` time (3 times by default) before waiting for `time-reopen()` time to try sending it again. +*Description:* This is an optional parameter. If message delivery fails, {{ site.product.short_name }} +retries sending the messages for retries() time (3 times by default) before waiting for `time-reopen()` time to try sending it again. ## topic() -| Type:| string| - |Default:| N/A| +| Type: | string | +|Default: | N/A | +|Mandatory:| yes | *Description:* The Kafka topic under which the message is published. You can use templates to change the topic dynamically based on the source or the content of -the message, for example, topic("${HOST}"). +the message, for example, `topic("${HOST}")`. >**NOTE:** Valid topic names for the topic() and fallback-topic() options have the >following limitations: -> +> >The topic name must contain characters within the pattern \[-._a-zA-Z0-9\]. -> +> >The length of the topic name must be between 1 and 249 characters. -> +> {: .notice--info} **NOTE:** If you use templates with the topic() option, configuring the @@ -249,6 +207,6 @@ option only if your Kafka clients have many threads and they do not receive enough messages. **NOTE:** Kafka clients have their own threadpool, entirely independent from -any {{ site.product.short_name }} settings. The workers() option has no effect on this +any {{ site.product.short_name }} settings. The `workers()` option has no effect on this threadpool. {: .notice--info} diff --git a/doc/_admin-guide/070_Destinations/100_Kafka-c/README.md b/doc/_admin-guide/070_Destinations/100_Kafka-c/README.md index d0bf4a13..40b1c09d 100644 --- a/doc/_admin-guide/070_Destinations/100_Kafka-c/README.md +++ b/doc/_admin-guide/070_Destinations/100_Kafka-c/README.md @@ -4,8 +4,7 @@ title: 'kafka(): Publishing messages to Apache Kafka using the short_title: kafka id: adm-dest-kafkac description: >- - Starting with version 3.21, {{ site.product.name }} (syslog-ng - OSE) can directly publish log messages to the Apache Kafka + Starting with version 3.21, {{ site.product.name }} can directly publish log messages to the Apache Kafka message bus, where subscribers can access them. --- From c99519893d927faa92f9645c80c2da2a8d337de9 Mon Sep 17 00:00:00 2001 From: Hofi Date: Thu, 27 Nov 2025 16:31:18 +0100 Subject: [PATCH 6/6] sources: started to add the missing common source options Signed-off-by: Hofi --- .../doc/admin-guide/options/chain-hostname.md | 6 ++++ .../admin-guide/options/default-priority.md | 13 ++++++-- .../doc/admin-guide/options/dns-cache.md | 6 ++++ .../060_Sources/020_File/README.md | 2 +- .../060_Sources/021_Wildcard-file/README.md | 2 +- .../038_Kafka/001_Kafka_options.md | 32 +++++++++++++++++++ .../085_macOS/000_darwin_oslog_options.md | 31 ++++++++++++++++-- .../000_otlp_source_options.md | 30 +++-------------- .../000_systemd-journal_options.md | 8 ----- 9 files changed, 90 insertions(+), 40 deletions(-) create mode 100644 _includes/doc/admin-guide/options/chain-hostname.md create mode 100644 _includes/doc/admin-guide/options/dns-cache.md diff --git a/_includes/doc/admin-guide/options/chain-hostname.md b/_includes/doc/admin-guide/options/chain-hostname.md new file mode 100644 index 00000000..2d9aa79a --- /dev/null +++ b/_includes/doc/admin-guide/options/chain-hostname.md @@ -0,0 +1,6 @@ +## chain-hostname() + +| Type:| `yes`, `no`| +|Default:| `no`| + +*Description:* This option can be used to enable or disable the chained hostname format. For more information, see the chain-hostnames() global option. diff --git a/_includes/doc/admin-guide/options/default-priority.md b/_includes/doc/admin-guide/options/default-priority.md index e015d797..15b8ceca 100644 --- a/_includes/doc/admin-guide/options/default-priority.md +++ b/_includes/doc/admin-guide/options/default-priority.md @@ -1,8 +1,15 @@ +## default-level() + +This is just an alias of default-priority(). + ## default-priority() | Type:| priority string | | Default: | {{ page.priority_default | default: '' }} | -*Description:* This parameter assigns an emergency level to the messages -received from the {{ page.src | default: 'file' }} source if the message does not specify one. For -example, default-priority(warning). +*Description:* This option defines the default level value if the `PRIORITY` entry does not exist in the msg received from the {{ page.src | default: 'file' }} source. +For example, `default-priority(warning)`. + +## default-severity() + +This is just an alias of default-priority(). diff --git a/_includes/doc/admin-guide/options/dns-cache.md b/_includes/doc/admin-guide/options/dns-cache.md new file mode 100644 index 00000000..f124bda0 --- /dev/null +++ b/_includes/doc/admin-guide/options/dns-cache.md @@ -0,0 +1,6 @@ +## dns-cache() + +| Type:| `yes`, `no`| +|Default:| `no`| + +*Description:* This option enables or disables the DNS cache usage. diff --git a/doc/_admin-guide/060_Sources/020_File/README.md b/doc/_admin-guide/060_Sources/020_File/README.md index 67262029..0bb2a290 100644 --- a/doc/_admin-guide/060_Sources/020_File/README.md +++ b/doc/_admin-guide/060_Sources/020_File/README.md @@ -48,6 +48,6 @@ source s_tail { **NOTE:** If the message does not have a proper syslog header, syslog-ng treats messages received from files as sent by the kern facility. Use -the **default-facility()** and **default-priority()** options in the +the default-facility() and default-priority() options in the source definition to assign a different facility if needed. {: .notice--info} diff --git a/doc/_admin-guide/060_Sources/021_Wildcard-file/README.md b/doc/_admin-guide/060_Sources/021_Wildcard-file/README.md index 1ce9b8a3..b8c37bce 100644 --- a/doc/_admin-guide/060_Sources/021_Wildcard-file/README.md +++ b/doc/_admin-guide/060_Sources/021_Wildcard-file/README.md @@ -50,7 +50,7 @@ Note the following important points: - If the message does not have a proper syslog header, {{ site.product.short_name }} treats messages received from files as sent by the user facility. - Use the **default-facility()** and **default-priority()** options in + Use the default-facility() and default-priority() options in the source definition to assign a different facility if needed. - For every message that {{ site.product.short_name }} reads from the source files, diff --git a/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md b/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md index 85a1ca7e..6eed8963 100644 --- a/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md +++ b/doc/_admin-guide/060_Sources/038_Kafka/001_Kafka_options.md @@ -1,6 +1,7 @@ --- title: "Options of the kafka() source" id: adm-src-kafka-opt +src: kafka description: >- This section describes the options of the kafka() source in {{ site.product.short_name }}. --- @@ -13,18 +14,32 @@ To use the kafka() source, the following two options are required: bootstrap-ser {% include doc/admin-guide/options/bootstrap-servers.md %} +{% include doc/admin-guide/options/chain-hostname.md %} + {% include doc/admin-guide/options/config-kafka.md kafka_type='consumer' type='source' protected_options='`bootstrap.servers` `metadata.broker.list` `enable.auto.offset.store` `auto.offset.reset` `enable.auto.commit` `auto.commit.enable`' %} +{% include doc/admin-guide/options/default-facility.md %} + +{% include doc/admin-guide/options/default-priority.md %} + {% include doc/admin-guide/options/disable-bookmarks.md %} See Bookmarking in the kafka() source for more details. +{% include doc/admin-guide/options/dns-cache.md %} + {% include doc/admin-guide/options/hook.md %} +{% include doc/admin-guide/options/host-override.md %} + {% include doc/admin-guide/options/ignore-saved-bookmarks.md %} (depending on the setting of the read-old-records() option.\ See Bookmarking in the kafka() source for more details. {% include doc/admin-guide/options/kafka-logging.md %} +{% include doc/admin-guide/options/keep-hostname.md %} + +{% include doc/admin-guide/options/keep-timestamp.md %} + ## log-fetch-limit() | Type: | integer | @@ -72,6 +87,8 @@ For example, with `workers(3)` and `fetch-limit(100000)`, the 2 processor worker {% include doc/admin-guide/options/read-old-records.md %}\ See Bookmarking in the kafka() source for more details. +{% include doc/admin-guide/options/program-override.md %} + ## separated-worker-queues() | Type: | yes \| no | @@ -82,6 +99,13 @@ See Bookmarking in the kafka() source for more details. **NOTE:** This approach can improve performance, especially in high-throughput scenarios, but may also lead to significantly increased memory usage. {: .notice--info} +## state-update-timeout() + +| Type: | integer in milliseconds | +|Default:| 1000 | + +*Description:* Specifies the maximum amount of time {{ site.product.short_name }} waits during Kafka broker state queries or other requests, such as metadata queries, partition offset queries/seeking, etc. + ## strategy-hint() | Accepted values: | assign, subscribe | @@ -93,6 +117,10 @@ Why is it worth using dual consumer strategies? describes the differences betwee For details about how the resulting topic names, partitions, and Kafka assign/subscribe strategies are determined in different scenarios, see Basic startegy usage cross-reference of the different topic configuration cases +{% include doc/admin-guide/options/tags.md %} + +{% include doc/admin-guide/options/time-zone.md %} + ## time-reopen() | Type: | integer in seconds | @@ -130,6 +158,10 @@ The partition number must be: For details about how the resulting topic names, partitions, and Kafka assign/subscribe strategies are determined in different scenarios, see Basic startegy usage cross-reference of the different topic configuration cases and Why is it worth using dual consumer strategies? +{% include doc/admin-guide/options/use-dns.md %} + +{% include doc/admin-guide/options/use-fqdn.md %} + ## workers() | Type: | integer | diff --git a/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md b/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md index fb53cf65..393fef88 100644 --- a/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md +++ b/doc/_admin-guide/060_Sources/085_macOS/000_darwin_oslog_options.md @@ -10,6 +10,16 @@ description: >- The `darwin-oslog()` source has the following options. +{% include doc/admin-guide/options/chain-hostname.md %} + +{% include doc/admin-guide/options/default-facility.md %} + +{% include doc/admin-guide/options/default-priority.md %} + +{% include doc/admin-guide/options/disable-bookmarks.md %} + +{% include doc/admin-guide/options/dns-cache.md %} + ## filter-predicate() |Type:| string| @@ -17,8 +27,6 @@ The `darwin-oslog()` source has the following options. *Description:* String for native macOS log message filtering using predicates. For example, the following predicate selects AirDrop logs: `subsystem=="com.apple.sharing" and category=="AirDrop"` -{% include doc/admin-guide/options/disable-bookmarks.md %} - ## go-reverse() |Type:| boolean| @@ -28,10 +36,18 @@ The `darwin-oslog()` source has the following options. {% include doc/admin-guide/options/hook.md %} +{% include doc/admin-guide/options/host-override.md %} + +{% include doc/admin-guide/options/hook.md %} + {% include doc/admin-guide/options/ignore-saved-bookmarks.md %} (depending on the setting of the go-reverse() and the read-old-records() options. {% include doc/admin-guide/options/deprecated-options.md old='do-not-use-bookmark()' new='ignore-saved-bookmarks()' %} +{% include doc/admin-guide/options/keep-hostname.md %} + +{% include doc/admin-guide/options/keep-timestamp.md %} + {% include doc/admin-guide/options/log-fetch-delays.md %} {% include doc/admin-guide/options/log-fetch-limit.md %} @@ -43,4 +59,15 @@ The `darwin-oslog()` source has the following options. *Description:* The maximum distance in seconds that a bookmark can point backward. That is, if {{ site.product.short_name }} is stopped for 10 minutes (600 seconds) and `max-bookmark-distance()` is set to `60`, then {{ site.product.short_name }} starts reading the logs from 60 seconds before the startup, missing 9 minutes (540 seconds) worth of logs. +{% include doc/admin-guide/options/program-override.md %} + {% include doc/admin-guide/options/read-old-records.md %} + +{% include doc/admin-guide/options/tags.md %} + +{% include doc/admin-guide/options/time-zone.md %} + +{% include doc/admin-guide/options/use-dns.md %} + +{% include doc/admin-guide/options/use-fqdn.md %} + diff --git a/doc/_admin-guide/060_Sources/175_syslog-otlp/000_otlp_source_options.md b/doc/_admin-guide/060_Sources/175_syslog-otlp/000_otlp_source_options.md index 2472d5ea..2731bd37 100644 --- a/doc/_admin-guide/060_Sources/175_syslog-otlp/000_otlp_source_options.md +++ b/doc/_admin-guide/060_Sources/175_syslog-otlp/000_otlp_source_options.md @@ -1,6 +1,8 @@ --- title: syslog-ng-otlp() source options id: adm-src-sng-otlp-opt +src: syslog-ng-otlp +priority_default: notice description: >- This section describes the options of the syslog-ng-otlp() source in {{ site.product.short_name }}. --- @@ -66,12 +68,7 @@ destination { }; ``` -## chain-hostname() - -| Type:| `yes`, `no`| -|Default:| `no`| - -*Description:* This option can be used to enable or disable the chained hostname format. For more information, see the chain-hostnames() global option. +{% include doc/admin-guide/options/chain-hostname.md %} {% include doc/admin-guide/options/channel-args.md %} @@ -82,28 +79,11 @@ destination { *Description:* This option configures the upper limit of in-flight gRPC requests per worker. It is advisd to set this value in the range of 10s or 100s when there are a high number of clients sending simultaneously. In an optimized solution, the number of `workers()` and `concurrent-requests()` is greater than or equal to the number of clients. However, this can cause an increase in memory usage. -## default-facility() - -| Type:| facility string| -|Default:| kern| - -*Description:* This option assigns a facility value to messages received from the file source if the message does not specify one. - -## default-level() - -| Type:| string| -|Default:| notice| - -*Description:* This option defines the default level value if the `PRIORITY` entry does not exist. +{% include doc/admin-guide/options/default-facility.md %} {% include doc/admin-guide/options/default-priority.md %} -## dns-cache() - -| Type:| `yes`, `no`| -|Default:| `no`| - -*Description:* This option enables or disables the DNS cache usage. +{% include doc/admin-guide/options/dns-cache.md %} ## ebpf() diff --git a/doc/_admin-guide/060_Sources/190_systemd-journal/000_systemd-journal_options.md b/doc/_admin-guide/060_Sources/190_systemd-journal/000_systemd-journal_options.md index e5cb6776..7dae7c89 100644 --- a/doc/_admin-guide/060_Sources/190_systemd-journal/000_systemd-journal_options.md +++ b/doc/_admin-guide/060_Sources/190_systemd-journal/000_systemd-journal_options.md @@ -14,14 +14,6 @@ The systemd-journal() driver has the following options: {% include doc/admin-guide/options/default-priority.md %} -## default-level() - -|Type: |string| -|Default: |notice| - -*Description:* The default level value if the `PRIORITY` entry does not -exist. - {% include doc/admin-guide/options/hook.md %} {% include doc/admin-guide/options/host-override.md %}