diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f34e9b..135f5b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: - main pull_request: + types: [review_requested, ready_for_review] workflow_dispatch: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a5b54e2..a9cbb91 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,6 +6,13 @@ on: types: - completed + pull_request: + types: [review_requested, ready_for_review] + + push: + branches: + - main + workflow_dispatch: jobs: diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml index 5d73ae3..b3a4f2a 100644 --- a/.github/workflows/pkg.yml +++ b/.github/workflows/pkg.yml @@ -1,11 +1,9 @@ name: Packaging on: - # push: - # branches: - # - main - # tags: - # - v* + push: + tags: + - v* # Triggering on PRs and arbitrary branch pushes is not enabled because most of the time only the CI build should be # triggered, not the packaging build. In cases where you want to test changes to this workflow this trigger enables diff --git a/Cargo.lock b/Cargo.lock index d0d6dc2..4f7730d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -240,9 +240,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -252,9 +252,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.14" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -322,9 +322,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.14" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -553,9 +553,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -752,9 +752,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "h2" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -814,9 +814,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -1015,9 +1015,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1059,9 +1059,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -1143,7 +1143,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ - "base64 0.21.6", + "base64 0.21.7", "hyper", "indexmap 1.9.3", "ipnet", @@ -1590,7 +1590,7 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64 0.21.6", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -1632,9 +1632,9 @@ dependencies = [ [[package]] name = "roto" -version = "0.2.0-rc0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0c1fc5208c034ca0a47f1b604e446ff5ef7ad725eb468f183ab64c61b5ed07" +checksum = "dc5e48d3c4cf45811b4e93ec359548434149eab05f32b67fe35b7a31628009e0" dependencies = [ "arc-swap", "bytes", @@ -1649,7 +1649,7 @@ dependencies = [ [[package]] name = "rotonda" -version = "0.1.0-rc0" +version = "0.1.0" dependencies = [ "allocator-api2", "anyhow", @@ -1704,9 +1704,9 @@ dependencies = [ [[package]] name = "rotonda-fsm" -version = "0.1.0-rc0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd072c7a361c58ec598b3238ae8848ac38e0573af239a9284c72f74c083cf27b" +checksum = "eac66721bf3780f38b671cf2ce75c810db54e8549dbc7444a2f89311e969862d" dependencies = [ "bytes", "log", @@ -1727,9 +1727,9 @@ dependencies = [ [[package]] name = "rotonda-store" -version = "0.3.0-rc0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ac90f7f855dd4a382add56a6b719c82c4efc96b1691d03de6f868fd1f449083" +checksum = "40b305bbb60bc1ff074a9ca68ed462625d7d6dd792211586f64515b1e55f652c" dependencies = [ "arc-swap", "crossbeam-epoch", @@ -1741,9 +1741,9 @@ dependencies = [ [[package]] name = "routecore" -version = "0.4.0-rc0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74e425676b9279297c62620b325bf9341e2594ad2d3a6edd82b0c19cf9c09c4" +checksum = "7c899795a3dcdfb5681b559bf38378851e837519a9302d4ff80ea90c3a88c82d" dependencies = [ "bytes", "chrono", @@ -1809,11 +1809,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -1913,7 +1913,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ - "base64 0.21.6", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -1982,9 +1982,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3b187f0231d56fe41bfb12034819dd2bf336422a5866de41bc3fec4b2e3883e8" dependencies = [ "serde", ] @@ -2380,9 +2380,9 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2474,9 +2474,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2484,9 +2484,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -2499,9 +2499,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -2511,9 +2511,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2521,9 +2521,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -2534,15 +2534,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index b1a0769..ccb6c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "rotonda" -version = "0.1.0-rc0" +version = "0.1.0" edition = "2021" rust-version = "1.71" authors = ["NLnet Labs "] -description = "A modular analytical BGP Engine" +description = "composable, programmable BGP engine" repository = "https://github.com/NLnetLabs/rotonda/" keywords = ["routing", "bgp"] categories = ["network-programming"] @@ -25,8 +25,8 @@ log-reroute = "0.1" pin-project-lite = "0.2" rand = "0.8" reqwest = { version = "0.11", default-features = false } -routecore = { version = "0.4.0-rc0", features = ["bgp", "bmp", "serde"] } -rotonda-fsm = "0.1.0-rc0" +routecore = { version = "0.4.0", features = ["bgp", "bmp", "serde"] } +rotonda-fsm = "0.1.0" sanitise-file-name = "1.0" serde = { version = "1.0", features = ["derive", "rc"] } serde_json = { version = "1.0", features = ["preserve_order"] } @@ -53,8 +53,8 @@ layout-rs = { version = "0.1" } mqtt = { version = "0.23.0", package = "rumqttc", default-features = false } non-empty-vec = { version = "0.2", features = ["serde"]} percent-encoding = "2.3" -roto = "0.2.0-rc0" -rotonda-store = "0.3.0-rc0" +roto = "0.2.0" +rotonda-store = "0.3.0" serde_with = "3" smallvec = { version = "1.11", features = ["const_generics", "const_new", "union"] } tokio-metrics = { version = "0.3", default-features = false } @@ -97,10 +97,11 @@ assets = [ ["README.md", "usr/share/doc/rotonda/", "644"], ["doc/rotonda.1", "usr/share/man/man1/rotonda.1", "644"], ["etc/rotonda.conf.system-service", "etc/rotonda/rotonda.conf", "644"], - ["etc/bgp-in-filter.roto", "etc/rotonda/filters/bgp-in-filter.roto", "644"], - ["etc/bmp-in-filter.roto", "etc/rotonda/filters/bmp-in-filter.roto", "644"], - ["etc/rib-in-post-filter.roto", "etc/rotonda/filters/rib-in-post-filter.roto", "644"], - ["etc/rib-in-pre-filter.roto", "etc/rotonda/filters/rib-in-pre-filter.roto", "644"] + ["etc/rotonda/rotonda.example.conf", "etc/rotonda/rotonda.example.conf", "644"], + ["etc/rotonda/filters/bgp-in-filter.roto", "etc/rotonda/filters/bgp-in-filter.roto", "644"], + ["etc/rotonda/filters/bmp-in-filter.roto", "etc/rotonda/filters/bmp-in-filter.roto", "644"], + ["etc/rotonda/filters/rib-in-post-filter.roto", "etc/rotonda/filters/rib-in-post-filter.roto", "644"], + ["etc/rotonda/filters/rib-in-pre-filter.roto", "etc/rotonda/filters/rib-in-pre-filter.roto", "644"] ] maintainer-scripts = "pkg/debian" changelog = "target/debian/changelog" # this will be generated by the pkg workflow @@ -110,7 +111,8 @@ conf-files = [ "/etc/rotonda/filters/bgp-in-filter.roto", "/etc/rotonda/filters/bmp-in-filter.roto", "/etc/rotonda/filters/rib-in-post-filter.roto", - "/etc/rotonda/filters/rib-in-pre-filter.roto" + "/etc/rotonda/filters/rib-in-pre-filter.roto", + "/etc/rotonda/rotonda.example.conf" ] systemd-units = { unit-name = "rotonda", unit-scripts = "pkg/common", enable = false } @@ -126,10 +128,11 @@ assets = [ { source = "target/rpm/rotonda.service", dest = "/lib/systemd/system/rotonda.service", mode = "644" }, { source = "doc/rotonda.1", dest = "/usr/share/man/man1/rotonda.1", mode = "644", doc = true }, { source = "etc/rotonda.conf.system-service", dest = "/etc/rotonda/rotonda.conf", mode = "644", config = true }, - { source = "etc/bgp-in-filter.roto", dest = "/etc/rotonda/filters/bgp-in-filter.roto", mode = "644", config = true }, - { source = "etc/bmp-in-filter.roto", dest = "/etc/rotonda/filters/bmp-in-filter.roto", mode = "644", config = true }, - { source = "etc/rib-in-post-filter.roto", dest = "/etc/rotonda/filters/rib-in-post-filter.roto", mode = "644", config = true }, - { source = "etc/rib-in-pre-filter.roto", dest = "/etc/rotonda/filters/rib-in-pre-filter.roto", mode = "644", config = true } + { source = "etc/rotonda/rotonda.example.conf", dest = "/etc/rotonda/rotonda.example.conf", mode = "644", config = true }, + { source = "etc/rotonda/filters/bgp-in-filter.roto", dest = "/etc/rotonda/filters/bgp-in-filter.roto", mode = "644", config = true }, + { source = "etc/rotonda/filters/bmp-in-filter.roto", dest = "/etc/rotonda/filters/bmp-in-filter.roto", mode = "644", config = true }, + { source = "etc/rotonda/filters/rib-in-post-filter.roto", dest = "/etc/rotonda/filters/rib-in-post-filter.roto", mode = "644", config = true }, + { source = "etc/rotonda/filters/rib-in-pre-filter.roto", dest = "/etc/rotonda/filters/rib-in-pre-filter.roto", mode = "644", config = true } ] # These get set using cargo-generate-rpm --set-metadata at package build time. #post_install_script = ... diff --git a/Changelog.md b/Changelog.md index 5e85e97..c558c14 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,23 +2,6 @@ ## 0.1.0 -Unreleased +Released 2024-01-19 - -Breaking changes - - -New - - -Bug fixes - - -Other changes - - -## 0.1.0-rc0 - -Released 2024-01-11. - -First release candidate. +First release diff --git a/ROADMAP.md b/ROADMAP.md index e73989e..32b92be 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,13 +5,19 @@ Roadmap - ✅ work item complete - 🦀 work item in progress - 💤 work item not started yet +- ↑ updated version General Features ================ -In order of priority: +RELEASED -MVP +rotonda 0.1 +rotonda-fsm 0.1 +roto 0.2 +routecore 0.4 +rotonda-store 0.3 +rotoro - - ✅ BMP listener - ✅ BMP proxy @@ -20,33 +26,45 @@ MVP - ✅ client-specific runtime application with BMP ingress and RIBs - ✅ `roto` filtering language - ✅ BGP passive speaker -- 🦀 daemonize +- ✅ limited systemd functionality - 🦀 ~~BGP limited active speaker~~ POSTPONED -- 🦀 Documentation -- 🦀 Packaging +- ✅ Documentation +- ✅ Packaging -NEXT RELEASES +NEXT RELEASE + +↑ rotonda 0.1.x + rotonda-fsm 0.1 +↑ roto 0.2.x + routecore 0.4 + rotonda-store 0.3 + rotoro - -Each item may constitute a new release +- 🦀 implement FilterMap (user-defined rx/tx and RIB storage type) +- 🦀 implement passive external data source: RIBs -- 🦀 ingress connectors (e.g. Kafka, MQTT) -- 🦀 egress connectors (e.g. Kafka, MQTT) -- 💤 create plugin system. -- 💤 `mrt` file connector from RIBs. -- 💤 egress connector to parquet. -- 🦀 query engine REST API -- 💤 query engine CLI -- 🦀 historical records storage + snapshotting ("3 weeks of data") -- 🦀 Distributed `rotonda`-`rotonda` setup with `rotoro` (experimental) +NEXT RELEASES + +- 💤 support more AFI/SAFIs, e.g. FlowSpec, L2VPN, MPLS +- 💤 emit BGP packets as events on OutputStream +- 💤 egress modified/created BGP packets (on peering sessions) +- 💤 RIB split over in-memory and on-disk +- 💤 refactor configuration: dynamic units reconfiguration with RESTCONF/yang+json +- 💤 implement active external data sources: RTR +- 💤 more ingress connectors (e.g. Kafka, MQTT, mrt) +- 💤 more egress connectors (e.g. Kafka, Parquet, mrt) +- 💤 create plugin system +- 💤 query engine over (split) RIBs. +- 💤 Rotonda shell +- 💤 Distributed `rotonda`-`rotonda` setup with `rotoro` (experimental) Development per Component ========================= - ## `Roto language` -MVP +RELEASED 0.2 - ✅ language lay-out - ✅ EBNF scheme @@ -54,88 +72,99 @@ MVP - ✅ create experimental compiler - ✅ create experimental virtual machine - ✅ implement all methods on `roto` types -- ✅ Integrate into `rotonda` -- 💤 create user-friendly error messaging -- 🦀 extensive stress testing -- 🦀 create high-level overview documentation +- ✅ integrate into `rotonda` +- ✅ implement FilterMap (user-defined rx/tx) +- 💤 ~create user-friendly error messaging~ +- 🦀 ~extensive stress testing +- 🦀 create manual-like docs +- ✅ create high-level overview documentation + +UNRELEASED 0.2.x + +- 🦀 complete passive external data sources for RIBS NEXT RELEASES -- 🦀 Simple passive data sources use cases +- 💤 implement more passive external data sources +- 💤 implement active external data sources: RTR +- 🦀 create reference-level documentation - 💤 create namespaces / modules system - 💤 create user-configurable graph DSL for units - 💤 create user-configurable query DSL - 💤 create dev documentation -- 💤 create manual-like docs -- 💤 create reference-level documentation ## `Rotonda-fsm` -MVP +RELEASE 0.1 - ✅ BMP state machine - ✅ BGP state machine - ✅ BGP passive speaker (session management) -- 🦀 ~BGP minimal active speaker~ -NEXT RELEASES +UNRELEASED 0.2 -- 💤 more BGP active speaker features +- 🦀 BGP active speaker ## `Routecore` -MVP +RELEASE 0.4 - ✅ prefix types - ✅ route record example types - ✅ BGP/BMP types for parsing -- 🦀 ~~create minimal BGP packet builder: Withdrawal for one prefix~~ -- 🦀 ~~create minimal BGP packet modifier: Add Community~~ + +UNRELEASED 0.5 + +- 🦀 create BGP packet builder +- 💤 partial FlowSpec parser NEXT RELEASES -- 💤 create BGP packet builder - 💤 BGPsec parser ## `Rotonda` -MVP +RELEASE 0.1 - ✅ setup `tokio` skeleton with logging etc. - ✅ setup BMP listener - ✅ REST API setup - ✅ MQTT OutputConnector -- ✅ BMP proxy - ✅ integrate BGP passive listener (BGP EventSource) - ✅ virtual RIB experimental implementation -- 🦀 ~integrate BGP limited speaker (BGP EventEmitter)~ -- 🦀 installation and usage documentation -- 🦀 limited packaging +- ✅ installation and usage documentation +- ✅ limited packaging +- ✅ limited `systemd` integration + +UNRELEASED 0.1.x + +- 🦀 passive external data sources: RIBs +- 🦀 implement FilterMap (user-defined rx/tx and RIB storage type) NEXT RELEASES -- 🦀 file OutputConnector -- 🦀 user-configurable RIBs experimental implementation -- 🦀 unit structure +- 💤 emit BGP packets as events on OutputStream +- 💤 egress modified/created BGP packets (on peering sessions) +- 💤 improved daemon functionality: drop privileges +- 🦀 refactor configuration: dynamic units reconfiguration with RESTCONF/yang+json - 🦀 implement tracing +- 💤 RIB split over in-memory and on-disk - 💤 namespaces / modules support +- 💤 more ingress connectors (e.g. Kafka, mrt) +- 💤 more egress connectors (e.g. Kafka, Parquet, mrt, (timescale) RDBMS) +- 🦀 BMP proxy - 💤 create experimental global registry -- 💤 Kafka connector -- 💤 split-off EventSource, OutputConnector units -- 💤 split-off EventEmitter, OutputConnector units -- 💤 time-series DB OutputConnector -- 💤 keep history window (serials) - 💤 snapshot/restore functionality -- 💤 `systemd` integration +- 💤 RIB diff functionality - 💤 create experimental distributed units ## `Rotoro` -NEXT RELEASES +MVP - 🦀 design wire protocol and select a layout (AVRO?) - 💤 create experimental de/serialization @@ -144,7 +173,7 @@ NEXT RELEASES ## `Rotonda-store` -MVP +0.3 - ✅ stabilize single-threaded store - ✅ stabilize multi-threaded store diff --git a/build.rs b/build.rs deleted file mode 100644 index 1546805..0000000 --- a/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -use std::env; - -fn main() { - println!( - "cargo:warning=Directory '/etc' can be found in {:?}", - env::var_os("CARGO_MANIFEST_DIR").unwrap() - ); -} diff --git a/etc/rotonda.conf b/etc/rotonda.conf deleted file mode 100644 index 142098c..0000000 --- a/etc/rotonda.conf +++ /dev/null @@ -1,902 +0,0 @@ -# Rotonda configuration file. -# -# This file must be in TOML format (https://toml.io/) and is structured as -# follows: -# -# global settings -# 1 or more units -# 1 or more targets -# -# Collectively units and targets are referred to as components. -# -# Data flows from West to East beginning with at least one input unit, through -# zero or more intermediate units and out terminating at at least one target. -# -# Additionally Rotonda has HTTP interfaces to the North and output stream -# interfaces to the South. The HTTP interfaces to the North may be used to -# inspect and interact with the application. Some types of units and target -# extend the HTTP interface with additional capabilities. The output stream -# interfaces to the South provide support for alternate forms of output such -# as MQTT event publication, logging/capture to file and proxying to external -# parties. -# -# Taken together one can think of the flow of information like so: -# -# (North) -# HTTP APIs -# ^ | -# | | -# | v -# (West) BGP/BMP inputs --> pipeline --> BGP/BMP outputs (East) -# | -# | -# v -# other outputs -# (South) -# -# Data can only be successfully passed from one component to another if the -# receiving component supports the value type output by the producing -# component. Consult the "Pipeline interaction" sections in the documentation -# below to ensure that your chosen inputs and outputs are compatible with each -# other. - - - - -### Introduction to components (units & targets) ############################# -# -# A unit is an input or intermediate processing stage. A target is a final -# output stage. There must always be at least one unit with one downstream -# target. -# -# Unit and target definitions have similar forms: -# -# [units.] [targets.] -# type = "" type = "" -# ... ... -# -# Names must be unique, types must be valid and any mandatory settings -# specific to the component type must be specified. -# -# The currently available components are intended to be used like so: -# -# bmp-tcp-in / bgp-tcp-in -> rib -> mqtt-out -# -# Additionally there are some components intended for diagnostic use: -# -# bmp-tcp-out, bmp-fs-out and null-out -# -# Each unit is able to process certain types of input and emit certain types -# of output. More information about each component type is given below. - - - - -### Introduction to Roto scripting ########################################### -# -# Some units and targets take one or more filter(-map) names which refer to -# "filter" or "filter-map" blocks in any loaded Roto script files. Roto files -# are written in the Rotonda Roto scripting language. Each file may contain -# a mix of "filter" and "filter-map" blocks. "filter" blocks accept or reject -# their Western input. "filter-map" blocks act the same but can also "map" the -# input from the West to a different output on the East. Both "filter" and -# "filter-map" blocks can also send data to one or more output streams to the -# South. -# -# Roto scripts work with Roto Types (RTs). All Roto script inputs, outputs and -# intermediate values are Roto Types. Different units and targets accept and -# produce different Roto Types and for a Rotonda pipeline to work properly -# input and output types must be correctly aligned. -# -# When Roto scripts send output to output streams to the South the data -# published to the stream is in the form of a Roto Record type which consists -# of key/value pairs, two of which have special meaning in Rotonda: -# -# - name: This key should have a string value which identifies the name of -# the target which is intended to handle the output Roto value. That -# target must still receive the value. -# - topic: This key should have a string value which may be used by a target -# that processes the output Roto value to determine what to do with -# it, e.g. in the case of the MQTT target it can be used to -# influence the eponymous MQTT topic to which a message will be -# published. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# roto_scripts_path A "path/to/a/directory/containing/*.roto" script -# (def: None) files. Each script file will be loaded & compiled -# and may be referred to in unit and target -# settings by using the name of a filter defined in -# the script file with the filter_name setting of -# the unit or target. -# -# Note: In the diagrams below the term "RT" denotes any valid Roto scripting -# type. - - - - -### HTTP API ################################################################# -# -# The HTTP API offers endpoints for interacting with and monitoring Rotonda at -# runtime: -# -# Endpoint Description -# ======================================================================== -# /metrics - Prometheus [1] metrics for monitoring Rotonda. -# See also the supplied example Grafana [2] -# dashboard file (`grafana-dashboard.json`). -# -# /status - Human readable application status information -# -# / - Some components (see below) offer their own HTTP -# API endpoints. -# -# [1]: https://prometheus.io/docs/introduction/overview/ -# [2]: https://prometheus.io/docs/visualization/grafana/ -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# http_listen The ":" to listen on for -# incoming HTTP requests. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# response_compression Whether or not to GZIP compress responses if the -# (def: true) client expresses support for it (via the HTTP -# "Accept-Encoding: gzip" request header). Set to -# false to completely disable GZIP response -# compression. - - - - -### Unit: bgp-tcp-in ############################################################# -# -# This unit listens on a specified TCP/IP address and port number for incoming -# connections from zero or more RFC 4271 [1] BGP speakers. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# listen The ":" to listen on for -# incoming BGP connections from BGP speakers. -# -# my_asn The positive number of the Autonomous System in -# which this instance of Rotonda is operating and -# which will be sent by this BGP speaker in its -# RFC 4271 BGP OPEN message in the "My Autonomous -# Number" field [3]. -# -# my_bgp_id An array of four positive integer numbers, e.g. -# [1, 2, 3, 4], which together define per RFC 4271 -# "A 4-octet unsigned integer that indicates the -# BGP Identifier of the sender of BGP messages" -# which is "determined upon startup and is the same -# for every local interface and BGP peer" [2]. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# peers."address" This setting define the set of peers from which -# (def: None) incoming connections will be accepted. By default -# no such peers are defined and thus all incoming -# connections are accepted. -# -# The double-quoted address value must be an IPv4 -# or IPv6 address or a prefix (an IP address and -# positive integer maximum length separated by a -# forward slash, e.g. "1.2.3.4/32"). -# -# The value of this setting is a TOML table which -# may be specified inline or as a separate section -# in the config file, e.g.: -# -# [units.my-bgp-in.peers.".."] -# name = .. -# remote_asn = .. -# -# Or: -# -# [units.my-bgp-in] -# peers.".." = { name = .., remote_asn = .. } -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every BGP UPDATE PDU -# received by this unit. If the script terminates -# with "reject" the UPDATE PDU will be discarded -# as if it had never been received. -# -# protocols The list of address families (AFI/SAFI) -# that is accepted from this peer. These are -# announced in the BGP OPEN as MultiProtocol -# Capabilities (RFC4760). In order to receive 'as -# much as possible', list all options. -# If this setting is omitted or set to the empty -# list, the session will only carry conventional IPv4 -# Unicast information. -# -# Currently supported are: -# Ipv4Unicast, Ipv6Unicast, -# Ipv4Multicast, Ipv6Multicast, -# Ipv4MplsUnicast, Ipv6MplsUnicast, -# Ipv4MplsVpnUnicast, Ipv6MplsVpnUnicast, -# Ipv4RouteTarget, -# Ipv4FlowSpec, Ipv6FlowSpec, -# L2VpnVpls, L2VpnEvpn -# -# addpath The list of address families (AFI/SAFI) for which -# ADDPATH Capabilities (RFC7911) will be announced in -# the BGP OPEN sent to this peer. If this setting is -# omitted or set to the empty list, no capabilities -# is announced. Supported address families are the -# same as listed for the 'protocols' setting above, -# though they do not make sense in all cases. -# -# -# The following MANDATORY settings MUST be specified in a peers."address" -# table: -# -# Setting Description -# ======================================================================== -# name A name identifying the remote peer intended to -# make it easier for the operator to know which -# BGP speaker these settings refer to. -# -# remote_asn The positive number, or [set, of, numbers], of -# the Autonomous System(s) which from which a -# remote BGP speaker that connects to this unit may -# identify itself (in the "My Autonomous Number" -# field of the RFC 4271 BGP OPEN message [3]) as -# belonging to. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +-------------------------------------------------+ -# TCP/IP --> | BgpUpdateMessage -> filter --> BgpUpdateMessage | --> N * Route -# +-----------------------|-------------------------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# One Route value is output per prefix announced or withdrawn via a BGP UPDATE -# message received. Withdrawals may also be synthesized if the BGP session is -# disconnected or the TCP/IP connection to the remote BGP speaker is lost. -# -# [1]: https://www.rfc-editor.org/rfc/rfc4271 -# [2]: https://www.rfc-editor.org/rfc/rfc4271#section-1.1 -# [3]: https://www.rfc-editor.org/rfc/rfc4271#section-4.2 - - - - -### Unit: bmp-tcp-in ######################################################### -# -# This unit implement an RFC 7854 "BGP Monitoring Protocol (BMP)" -# "monitoring station" [1] by listening on a specified TCP/IP address and port -# number for incoming connections from zero or more BMP capable routers. This -# unit processes the incoming raw BMP messages through a BMP state machine in -# order to extract, store and propagate downstream the route announcements and -# withdrawals. -# -# This unit extends the HTTP API with endpoints that output HTML and text -# formatted information about the monitored routers currently streaming data -# into Rotonda. These endpoints are intended for operators as a diagnostic aid -# and not for automation purposes. The output format is not intended to be -# machine readable and may change without warning. -# -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# listen The ":" to listen on for -# incoming BGP connections from BGP speakers. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# http_api_path The relative URL prefix for HTTP REST API calls -# (def: /routers/) responded to by this instance of this unit. -# -# router_id_template A user defined "" that is used to name -# (def: {sys_name}) incoming router connections according to a user -# supplied template which may include the following -# placeholders which will be expanded into their -# respective values for the monitored router. -# -# {sys_name} - Router RFC 7854 sysName. -# {router_ip} - Router source IP address. -# {router_port} - Router source port. -# -# Note: {sys_name} will be "unknown" until the -# sysName information TLV is received from the -# router as part of the BMP Initiation Message that -# it is required to send before any other messages. -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every BMP message -# received by this unit. If the script terminates -# with "reject" the BMP message will be discarded. -# as if it had never been received. -# -# tracing_mode Whether and how to trace BMP messages through the -# (def: Off) pipeline. -# -# When set to "On" all received BMP messages will -# be traced into successive tracing buffers -# numbered 0-255 inclusive. These can be seen on -# the status graph at: -# -# /status/graph/traces/N. -# -# When set to "IfRequested" received BMP messages -# whose upper niblle of the "Version" header byte -# is non-zero will cause that unsigned integer -# number to be used as the tracing buffer index to -# capture traces into. -# -# --- HTTP API --------------------------------------------------------------- -# -# REQUESTS: -# -# GET /routers/ -# GET /routers/ -# -# DESCRIPTION: -# -# This endpoint outputs information about the specified router if it is -# currently connected to the unit. -# -# PARAMETERS: -# -# Parameter Description -# ======================================================================== -# /routers/ Base path. Use the unit setting -# to change this if using multiple instances of -# this unit. -# -# The id of the router to query information about. -# Three different forms of router ID are supported: -# -# - :, OR -# - , OR -# - -# -# RESPONSE: GET /routers/ -# -# A HTML table showing all currently monitored routers and some basic -# information about them. -# -# RESPONSE: GET /routers/ -# -# A detailed plain text report about the monitored router and its -# interactions with Rotonda. -# -# --- Pipeline interaction --------------------------------------------------- -# -# HTTP API -# ^ | -# | | -# | v -# +-------------------------------------+ -# TCP/IP --> | BmpMessage -> filter --> BmpMessage | --> N * Route -# +-----------------|-------------------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# One Route value is output per prefix announced or withdrawn via a BGP UPDATE -# message received as the payload of a BMP Route Monitoring message. -# Withdrawals may also be synthesized due to BMP Peer Down notification or -# loss of TCP/IP connection to the monitored BMP router. -# -# [1]: https://www.rfc-editor.org/rfc/rfc7854 - - - - -### Unit: filter ############################################################# -# -# This unit runs a filter script that can be either a filter or a filter-map: -# -# - A filter accepts or rejects the input Roto value that it receives. -# - A filter-map does the same but the output Roto value can be different -# than the input value, i.e. as if the input was "mapped" to the output. -# - Both filter and filter-map scripts can optionally emit additional Roto -# values for consumption by particular targets. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every pipeline payload -# received by this unit. If the script terminates -# with "reject" the payload item will be discarded. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +--------+ -# RT --> | filter | --> Accept with RT, or Reject -# +--------+ -# | -# | -# v -# 0..N output streams each -# emitting values of a single RT - - - - -### Unit: rib ################################################################ -# -# This unit is a general purpose prefix store but is primarily intended to map -# prefixes to the details of the routes to those prefixes and the source from -# which they were received. -# -# It offers a HTTP API for querying the set of known routes to a longest match -# to a given IP prefix address and length. -# -# Upstream announcements cause routes to be added to the store. Upstream -# withdrawals cause routes to be flagged as withdrawn in the store. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# http_api_path The relative URL prefix for HTTP REST API calls -# (def: /prefixes/) responded to by this instance of this unit. -# -# query_limits.more_specifics.shortest_prefix_ipv4 (def: 8) -# query_limits.more_specifics.shortest_prefix_ipv6 (def: 19) -# These two settings protect against overly broad -# queries that require more time to lookup longest -# matching prefixes in the store. Queries for IPv4 -# prefixes shorter than /8 (e.g. /7), or for IPv6 -# prefixes shorter than /19 (e.g. /18), will result -# in a HTTP 400 Bad Request status code. -# -# rib_keys (def: ["PeerIp", "PeerAsn", "AsPath"]) -# Adjust this setting to control when routes are -# considered to be from the same peer and thus when -# that peer announces a route does it update or is -# in addition to an existing announcement, or when -# that peer withdraws a route, or its routes are -# withdrawn because the connection to it is lost, -# that only announced routes whose specified key -# fields match those of the withdrawal will be -# marked as withdrawn. Incorrectly specifying the -# set of key fields can lead to a different set of -# announced routes stored in the rib than expected. -# -# filter_name Either a single name of a loaded "filter" or -# "filter-map", or an ["array", "of", "filter", -# "or", "filter-map", "names"]. If more than one -# is specified, the additional entries cause -# virtual RIB units to be created to the East of -# this unit, each subsequent virtual RIB being -# further to the East than the last. The input type -# received by each "filter" or "filter-map" depends -# on the output type of the previous RIB unit. Each -# vRIB exposes its own HTTP REST API endpoint at -# {http_api_path}/{n}/ where {n} is zero for the -# first vRIB, 1 for the second vRIB, and so on. -# -# --- Pipeline interaction --------------------------------------------------- -# -# In summary the flow looks like this: -# -# RT --> filter1 --> pRIB --> filter2 --> vRIB1 --> filter2 --> vRIB2 --> .. -# -# Now lets break down the various different possible scenarios into more -# detail: -# -# 1. A single physical RIB with no Roto script filtering: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> | pRIB | --> RT -# +------+ -# -# -# -# 2. A single physical RIB with a Roto script filter: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> filter --> RT --> | pRIB | --> RT -# | +------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# -# 3. A physical RIB and a virtual RIB, each with their own Roto script filter: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> filter --> RT --> | pRIB | --> RT -->+ -# | +------+ | -# | | -# v | -# 0..N output streams each | -# emitting values of a single RT | -# v -# +<----------------------------------------+ -# | -# | HTTP API -# | ^ | -# | | | -# | | v -# v +------+ -# +--> filter --> RT --> | vRIB | --> RT -# | +------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# -# Some notes to be aware of: -# -# - Queries to the HTTP API of a virtual RIB are submitted upstream to the -# physical RIB and the results flow back down the pipeline to the -# requesting virtual RIB and out via its HTTP API. Results are processed -# through each vRIB filter yielding the vRIB modified "view" of the result -# data. -# -# ====================================================================== -# NOTE: Values emitted by output streams of vRIB filters when processing -# HTTP API query results are silently discarded, i.e. values emitted by -# output streams of vRIB filters are only honoured for input data that -# originated to the West of the pRIB, NOT for data that was the result -# of a HTTP API query. -# ====================================================================== -# -# - The input to a physical RIB is usually a Route but can also be a Record -# with a "prefix" key, but only Route values support the notion of being -# "withdrawn". The entire record (all its keys and values) will be added -# to the set of values stored at the prefix in the RIB, with the rib_keys -# fields determining whether a new value is added to the set or replaces -# an existing item in the set. - - - - -### Target: mqtt-out ######################################################### -# -# This target publishes JSON events to an MQTT broker via a TCP connection. -# -# NOTE: The MQTT broker is not part of Rotonda, it is a separate service that -# must be deployed and operated separately to Rotonda. -# -# Tested with the EMQX MQTT broker with both the free public MQTT 5 Broker [1] -# and with the EMQX Docker image [2]. -# -# This target ONLY accepts input data that: -# -# - Was received from a configured upstream source unit. -# - Was emitted by a Roto script output stream. -# - Is of type Record with a "name" field whose value matches the name of -# this instance of the mqtt-out target. -# -# So naming an instance of this unit in a Roto script output stream record is -# not sufficient to have this unit receive it, this unit must still be -# downstream of the producing unit to receive its output. -# -# The JSON event structure produced by this target is a direct serialization -# of the received Roto type as JSON, i.e. a record with a set of key/value -# pairs. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# destination A "host:port" string specifying the host or IP -# address of an MQTT broker to connect to. If the -# ":port" part is omitted the IANA registered MQTT -# port number [3] 1883 will be used. Note: Only -# unencrypted TCP connections are supported, i.e. -# TLS and WS are not supported. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# client_id A unique name to identify the client to the -# (def: "") server in order to hold state about the session. -# If empty the server will use a clean session and -# assign a random name to the client. Servers are -# required to support names upto 23 bytes in length -# but may support more. -# -# qos MQTT quality-of-service setting for determining -# (def: 2) how many times a message can be delivered: -# -# 0 (at most once) -# 1 (at least once) -# 2 (exactly once) -# -# Higher values require more synchronization with -# the broker leading to lower throughput but -# greater reliability/correctness. -# -# queue_size The number of messages that can be buffered for -# (def: 1000) delivery to the MQTT broker. -# -# connect_retry_secs The number of seconds to wait before attempting -# (def: 60) to reconnect to the MQTT broker if the connection -# is lost. -# -# publish_max_secs The number of seconds to wait before timing out -# (def: 5) an attempt to publish a message to the MQTT -# broker. -# -# topic_template A "string" template that will be used to -# (def: "rotonda/{id}") determine the MQTT topic to which events will be -# published. If present, the "{id}" placeholder -# will be replaced by the "topic" value in the -# incoming Record value. When using "{id}" an MQTT -# client that supports MQTT wildcards can still -# receive all events by subscribing to 'rotonda/#' -# for example. -# -# username A "string" username for login to the MQTT broker. -# -# password A "string" password for login to the MQTT broker. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +----------+ -# Record --> | mqtt-out | - - JSON - - > MQTT server -# +----------+ -# -# [1]: https://www.emqx.com/en/mqtt/public-mqtt5-broker -# [2]: https://hub.docker.com/r/emqx/emqx -# [3]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=mqtt - - - - -### Target: null-out ######################################################### -# -# This target discards everything it receives. -# -# Rotonda requires that there always be at least one target. Using this target -# allows you to run Rotonda for testing purposes without any "real" targets, -# or if the only output is via Roto script output stream messages. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# source The upstream unit from which data will be -# received. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +----------+ -# RT --> | null-out | -# +----------+ - - - - -### Target: bmp-fs-out ####################################################### -# -# This target writes raw BMP messages to files on disk, either separated per -# monitored router or merged into a single file. BMP messages can be written -# in one of three different formats: -# -# log, raw, or pcap text -# -# The log format is a limited one line per BMP message plain text log of BMP -# messages received and some limited information about each one. This format -# is intended for gaining a quick insight into the messages being received by -# Rotonda. -# -# The raw format writes the received BMP bytes out as-is, with each BMP -# message byte sequence preceeded by a number indicating how many BMP message -# bytes follow. This format is intended for capturing messages for replay for -# testing purposes later. -# -# THe PCAP text format can be transformed by the separate text2pcap tool, and -# from there can be viewed and analyzed using a tool like WireShark. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# source The upstream unit from which data will be -# received. -# -# path The path to which files will be written. If mode -# is "split" (the default) this setting specifies -# a directory (which must already exist) under -# which one file per router will be created. -# -# format Choose one of: "log", "raw", or "pcaptext". -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# mode Either "merge" or "split". In merged mode all BMP -# (def: "split") messages from all monitored routers are appended -# to the same file. In split mode separate output -# files will be written under an EXISTING directory -# specified by the "path" setting. In "merge" mode -# all BMP messages from all monitored routers will -# be written into a single file file defined by the -# "path" setting. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +------------+ -# BmpMessage --> | bmp-fs-out | - - - - > Writes to disk -# +------------+ - - - - -### Target: bmp-tcp-out ###################################################### -# -# This target writes raw BMP messages over a TCP connection to a specified -# destination IP address and port number, for instance to a second instance of -# Rotonda. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# destination A TCP IP address and port number to proxy raw BMP -# messages to. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# accept Zero or more "IP address" values defining routers -# (def: []) whose BMP messages will be proxied. If specified, -# ONLY the specified routers will be proxied, all -# others will be able to connect and send messages -# to Rotonda. -# -# reject Zero or more "IP address" values defining routers -# (def: []) whose BMP messages will NOT be proxied. If -# specified and "accept" is NOT specified, this -# setting will permit all other monitored routers -# BMP messages to be proxied. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +-------------+ -# BmpMessage --> | bmp-tcp-out | - - - - > Proxied via TCP/IP -# +-------------+ - - - - -### Sample configuration ##################################################### - -log_level = "info" # "error", "warn", "info", "debug" or "trace" -log_target = "stderr" # "stderr", "file" or "syslog" -log_facility = "daemon" # used if log_target is "syslog" -log_file = "./rotonda.log" # used if log_target is "file" - -# Optional path to a directory from which any .roto files present will be -# loaded into Rotonda. Filter names used below refer to blocks in the .roto -# files loaded from this directory. -roto_scripts_path = "etc/" - -http_listen = ["127.0.0.1:8080"] - -[units.bgp-in] -type = "bgp-tcp-in" -listen = "0.0.0.0:11179" # TODO: When we can drop privileges make this 179 -my_asn = 64512 # Tip: edit me, or comment this unit out -my_bgp_id = [0, 0, 0, 0] # Tip: edit me, or comment this unit out -filter_name = "bgp-in-filter" - -[units.bmp-in] -type = "bmp-tcp-in" -listen = "0.0.0.0:11019" -filter_name = "bmp-in-filter" -http_api_path = "/bmp-routers/" -tracing_mode = "Off" - -# The two RIB units below could have been defined like so: -# -# [units.rib-in-pre] -# type = "rib" -# sources = ["bgp-in", "bmp-in"] -# filter_name = ["rib-in-pre", "rib-in-post"] -# -# But then we don't control the unit naming or HTTP API paths, instead the -# virtual RIB would be named "rib-in-pre-vRIB-0" and its HTTP API path would -# be that of the pRIB with /0/ appended. - -[units.rib-in-pre] -type = "rib" -sources = ["bgp-in", "bmp-in"] -rib_type = "Physical" -filter_name = "rib-in-pre-filter" -http_api_path = "/rib-in-pre/" - -[units.rib-in-post] -type = "rib" -sources = ["rib-in-pre"] -rib_type = "Virtual" -vrib_upstream = "rib-in-pre" -filter_name = "rib-in-post-filter" -http_api_path = "/rib-in-post/" - -[targets.null] -type = "null-out" -sources = ["rib-in-post"] - -[targets.bmp-proxy] -type = "bmp-tcp-out" -sources = ["bmp-in"] -destination = "SOME_IP_ADDRESS:SOME_PORT" # Tip: edit me, or comment this target out - -[targets.mqtt] -type = "mqtt-out" -sources = ["bmp-in", "bgp-in", "rib-in-pre", "rib-in-post"] -client_id = "rotonda" # Tip: edit me -topic_template = "{id}" # Tip: edit me -destination = "SOME_IP_ADDRESS:SOME_PORT" # Tip: edit me, or comment this target out diff --git a/etc/rotonda.conf.system-service b/etc/rotonda.conf.system-service index dafbb85..c50025b 100644 --- a/etc/rotonda.conf.system-service +++ b/etc/rotonda.conf.system-service @@ -1,854 +1,22 @@ -# Rotonda configuration file. -# -# This file must be in TOML format (https://toml.io/) and is structured as -# follows: -# -# global settings -# 1 or more units -# 1 or more targets -# -# Collectively units and targets are referred to as components. -# -# Data flows from West to East beginning with at least one input unit, through -# zero or more intermediate units and out terminating at at least one target. -# -# Additionally Rotonda has HTTP interfaces to the North and output stream -# interfaces to the South. The HTTP interfaces to the North may be used to -# inspect and interact with the application. Some types of units and target -# extend the HTTP interface with additional capabilities. The output stream -# interfaces to the South provide support for alternate forms of output such -# as MQTT event publication, logging/capture to file and proxying to external -# parties. -# -# Taken together one can think of the flow of information like so: -# -# (North) -# HTTP APIs -# ^ | -# | | -# | v -# (West) BGP/BMP inputs --> pipeline --> BGP/BMP outputs (East) -# | -# | -# v -# other outputs -# (South) -# -# Data can only be successfully passed from one component to another if the -# receiving component supports the value type output by the producing -# component. Consult the "Pipeline interaction" sections in the documentation -# below to ensure that your chosen inputs and outputs are compatible with each -# other. - - - - -### Introduction to components (units & targets) ############################# -# -# A unit is an input or intermediate processing stage. A target is a final -# output stage. There must always be at least one unit with one downstream -# target. -# -# Unit and target definitions have similar forms: -# -# [units.] [targets.] -# type = "" type = "" -# ... ... -# -# Names must be unique, types must be valid and any mandatory settings -# specific to the component type must be specified. -# -# The currently available components are intended to be used like so: -# -# bmp-tcp-in / bgp-tcp-in -> rib -> mqtt-out -# -# Additionally there are some components intended for diagnostic use: -# -# bmp-tcp-out, bmp-fs-out and null-out -# -# Each unit is able to process certain types of input and emit certain types -# of output. More information about each component type is given below. - - - - -### Introduction to Roto scripting ########################################### -# -# Some units and targets take one or more filter(-map) names which refer to -# "filter" or "filter-map" blocks in any loaded Roto script files. Roto files -# are written in the Rotonda Roto scripting language. Each file may contain -# a mix of "filter" and "filter-map" blocks. "filter" blocks accept or reject -# their Western input. "filter-map" blocks act the same but can also "map" the -# input from the West to a different output on the East. Both "filter" and -# "filter-map" blocks can also send data to one or more output streams to the -# South. -# -# Roto scripts work with Roto Types (RTs). All Roto script inputs, outputs and -# intermediate values are Roto Types. Different units and targets accept and -# produce different Roto Types and for a Rotonda pipeline to work properly -# input and output types must be correctly aligned. -# -# When Roto scripts send output to output streams to the South the data -# published to the stream is in the form of a Roto Record type which consists -# of key/value pairs, two of which have special meaning in Rotonda: -# -# - name: This key should have a string value which identifies the name of -# the target which is intended to handle the output Roto value. That -# target must still receive the value. -# - topic: This key should have a string value which may be used by a target -# that processes the output Roto value to determine what to do with -# it, e.g. in the case of the MQTT target it can be used to -# influence the eponymous MQTT topic to which a message will be -# published. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# roto_scripts_path A "path/to/a/directory/containing/*.roto" script -# (def: None) files. Each script file will be loaded & compiled -# and may be referred to in unit and target -# settings by using the name of a filter defined in -# the script file with the filter_name setting of -# the unit or target. -# -# Note: In the diagrams below the term "RT" denotes any valid Roto scripting -# type. - - - - -### HTTP API ################################################################# -# -# The HTTP API offers endpoints for interacting with and monitoring Rotonda at -# runtime: -# -# Endpoint Description -# ======================================================================== -# /metrics - Prometheus [1] metrics for monitoring Rotonda. -# See also the supplied example Grafana [2] -# dashboard file (`grafana-dashboard.json`). -# -# /status - Human readable application status information -# -# / - Some components (see below) offer their own HTTP -# API endpoints. -# -# [1]: https://prometheus.io/docs/introduction/overview/ -# [2]: https://prometheus.io/docs/visualization/grafana/ -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# http_listen The ":" to listen on for -# incoming HTTP requests. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# response_compression Whether or not to GZIP compress responses if the -# (def: true) client expresses support for it (via the HTTP -# "Accept-Encoding: gzip" request header). Set to -# false to completely disable GZIP response -# compression. - - - - -### Unit: bgp-tcp-in ############################################################# -# -# This unit listens on a specified TCP/IP address and port number for incoming -# connections from zero or more RFC 4271 [1] BGP speakers. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# listen The ":" to listen on for -# incoming BGP connections from BGP speakers. -# -# my_asn The positive number of the Autonomous System in -# which this instance of Rotonda is operating and -# which will be sent by this BGP speaker in its -# RFC 4271 BGP OPEN message in the "My Autonomous -# Number" field [3]. -# -# my_bgp_id An array of four positive integer numbers, e.g. -# [1, 2, 3, 4], which together define per RFC 4271 -# "A 4-octet unsigned integer that indicates the -# BGP Identifier of the sender of BGP messages" -# which is "determined upon startup and is the same -# for every local interface and BGP peer" [2]. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# peers."address" This setting define the set of peers from which -# (def: None) incoming connections will be accepted. By default -# no such peers are defined and thus all incoming -# connections are accepted. -# -# The double-quoted address value must be an IPv4 -# or IPv6 address or a prefix (an IP address and -# positive integer maximum length separated by a -# forward slash, e.g. "1.2.3.4/32"). -# -# The value of this setting is a TOML table which -# may be specified inline or as a separate section -# in the config file, e.g.: -# -# [units.my-bgp-in.peers.".."] -# name = .. -# remote_asn = .. -# -# Or: -# -# [units.my-bgp-in] -# peers.".." = { name = .., remote_asn = .. } -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every BGP UPDATE PDU -# received by this unit. If the script terminates -# with "reject" the UPDATE PDU will be discarded -# as if it had never been received. -# -# protocols The list of address families (AFI/SAFI) -# that is accepted from this peer. These are -# announced in the BGP OPEN as MultiProtocol -# Capabilities (RFC4760). In order to receive 'as -# much as possible', list all options. -# If this setting is omitted or set to the empty -# list, the session will only carry conventional IPv4 -# Unicast information. -# -# Currently supported are: -# Ipv4Unicast, Ipv6Unicast, -# Ipv4Multicast, Ipv6Multicast, -# Ipv4MplsUnicast, Ipv6MplsUnicast, -# Ipv4MplsVpnUnicast, Ipv6MplsVpnUnicast, -# Ipv4RouteTarget, -# Ipv4FlowSpec, Ipv6FlowSpec, -# L2VpnVpls, L2VpnEvpn -# -# addpath The list of address families (AFI/SAFI) for which -# ADDPATH Capabilities (RFC7911) will be announced in -# the BGP OPEN sent to this peer. If this setting is -# omitted or set to the empty list, no capabilities -# is announced. Supported address families are the -# same as listed for the 'protocols' setting above, -# though they do not make sense in all cases. -# -# -# The following MANDATORY settings MUST be specified in a peers."address" -# table: -# -# Setting Description -# ======================================================================== -# name A name identifying the remote peer intended to -# make it easier for the operator to know which -# BGP speaker these settings refer to. -# -# remote_asn The positive number, or [set, of, numbers], of -# the Autonomous System(s) which from which a -# remote BGP speaker that connects to this unit may -# identify itself (in the "My Autonomous Number" -# field of the RFC 4271 BGP OPEN message [3]) as -# belonging to. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +-------------------------------------------------+ -# TCP/IP --> | BgpUpdateMessage -> filter --> BgpUpdateMessage | --> N * Route -# +-----------------------|-------------------------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# One Route value is output per prefix announced or withdrawn via a BGP UPDATE -# message received. Withdrawals may also be synthesized if the BGP session is -# disconnected or the TCP/IP connection to the remote BGP speaker is lost. -# -# [1]: https://www.rfc-editor.org/rfc/rfc4271 -# [2]: https://www.rfc-editor.org/rfc/rfc4271#section-1.1 -# [3]: https://www.rfc-editor.org/rfc/rfc4271#section-4.2 - - - - -### Unit: bmp-tcp-in ######################################################### -# -# This unit implement an RFC 7854 "BGP Monitoring Protocol (BMP)" -# "monitoring station" [1] by listening on a specified TCP/IP address and port -# number for incoming connections from zero or more BMP capable routers. This -# unit processes the incoming raw BMP messages through a BMP state machine in -# order to extract, store and propagate downstream the route announcements and -# withdrawals. -# -# This unit extends the HTTP API with endpoints that output HTML and text -# formatted information about the monitored routers currently streaming data -# into Rotonda. These endpoints are intended for operators as a diagnostic aid -# and not for automation purposes. The output format is not intended to be -# machine readable and may change without warning. -# -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# listen The ":" to listen on for -# incoming BGP connections from BGP speakers. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# http_api_path The relative URL prefix for HTTP REST API calls -# (def: /routers/) responded to by this instance of this unit. -# -# router_id_template A user defined "" that is used to name -# (def: {sys_name}) incoming router connections according to a user -# supplied template which may include the following -# placeholders which will be expanded into their -# respective values for the monitored router. -# -# {sys_name} - Router RFC 7854 sysName. -# {router_ip} - Router source IP address. -# {router_port} - Router source port. -# -# Note: {sys_name} will be "unknown" until the -# sysName information TLV is received from the -# router as part of the BMP Initiation Message that -# it is required to send before any other messages. -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every BMP message -# received by this unit. If the script terminates -# with "reject" the BMP message will be discarded. -# as if it had never been received. -# -# tracing_mode Whether and how to trace BMP messages through the -# (def: Off) pipeline. -# -# When set to "On" all received BMP messages will -# be traced into successive tracing buffers -# numbered 0-255 inclusive. These can be seen on -# the status graph at: -# -# /status/graph/traces/N. -# -# When set to "IfRequested" received BMP messages -# whose upper niblle of the "Version" header byte -# is non-zero will cause that unsigned integer -# number to be used as the tracing buffer index to -# capture traces into. -# -# --- HTTP API --------------------------------------------------------------- -# -# REQUESTS: -# -# GET /routers/ -# GET /routers/ -# -# DESCRIPTION: -# -# This endpoint outputs information about the specified router if it is -# currently connected to the unit. -# -# PARAMETERS: -# -# Parameter Description -# ======================================================================== -# /routers/ Base path. Use the unit setting -# to change this if using multiple instances of -# this unit. -# -# The id of the router to query information about. -# Three different forms of router ID are supported: -# -# - :, OR -# - , OR -# - -# -# RESPONSE: GET /routers/ -# -# A HTML table showing all currently monitored routers and some basic -# information about them. -# -# RESPONSE: GET /routers/ -# -# A detailed plain text report about the monitored router and its -# interactions with Rotonda. -# -# --- Pipeline interaction --------------------------------------------------- -# -# HTTP API -# ^ | -# | | -# | v -# +-------------------------------------+ -# TCP/IP --> | BmpMessage -> filter --> BmpMessage | --> N * Route -# +-----------------|-------------------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# One Route value is output per prefix announced or withdrawn via a BGP UPDATE -# message received as the payload of a BMP Route Monitoring message. -# Withdrawals may also be synthesized due to BMP Peer Down notification or -# loss of TCP/IP connection to the monitored BMP router. -# -# [1]: https://www.rfc-editor.org/rfc/rfc7854 - - - - -### Unit: filter ############################################################# -# -# This unit runs a filter script that can be either a filter or a filter-map: -# -# - A filter accepts or rejects the input Roto value that it receives. -# - A filter-map does the same but the output Roto value can be different -# than the input value, i.e. as if the input was "mapped" to the output. -# - Both filter and filter-map scripts can optionally emit additional Roto -# values for consumption by particular targets. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# filter_name The name of a loaded "filter" or "filter-map" -# that will be executed for every pipeline payload -# received by this unit. If the script terminates -# with "reject" the payload item will be discarded. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +--------+ -# RT --> | filter | --> Accept with RT, or Reject -# +--------+ -# | -# | -# v -# 0..N output streams each -# emitting values of a single RT - - - - -### Unit: rib ################################################################ -# -# This unit is a general purpose prefix store but is primarily intended to map -# prefixes to the details of the routes to those prefixes and the source from -# which they were received. -# -# It offers a HTTP API for querying the set of known routes to a longest match -# to a given IP prefix address and length. -# -# Upstream announcements cause routes to be added to the store. Upstream -# withdrawals cause routes to be flagged as withdrawn in the store. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# http_api_path The relative URL prefix for HTTP REST API calls -# (def: /prefixes/) responded to by this instance of this unit. -# -# query_limits.more_specifics.shortest_prefix_ipv4 (def: 8) -# query_limits.more_specifics.shortest_prefix_ipv6 (def: 19) -# These two settings protect against overly broad -# queries that require more time to lookup longest -# matching prefixes in the store. Queries for IPv4 -# prefixes shorter than /8 (e.g. /7), or for IPv6 -# prefixes shorter than /19 (e.g. /18), will result -# in a HTTP 400 Bad Request status code. -# -# rib_keys (def: ["PeerIp", "PeerAsn", "AsPath"]) -# Adjust this setting to control when routes are -# considered to be from the same peer and thus when -# that peer announces a route does it update or is -# in addition to an existing announcement, or when -# that peer withdraws a route, or its routes are -# withdrawn because the connection to it is lost, -# that only announced routes whose specified key -# fields match those of the withdrawal will be -# marked as withdrawn. Incorrectly specifying the -# set of key fields can lead to a different set of -# announced routes stored in the rib than expected. -# -# filter_name Either a single name of a loaded "filter" or -# "filter-map", or an ["array", "of", "filter", -# "or", "filter-map", "names"]. If more than one -# is specified, the additional entries cause -# virtual RIB units to be created to the East of -# this unit, each subsequent virtual RIB being -# further to the East than the last. The input type -# received by each "filter" or "filter-map" depends -# on the output type of the previous RIB unit. Each -# vRIB exposes its own HTTP REST API endpoint at -# {http_api_path}/{n}/ where {n} is zero for the -# first vRIB, 1 for the second vRIB, and so on. -# -# --- Pipeline interaction --------------------------------------------------- -# -# In summary the flow looks like this: -# -# RT --> filter1 --> pRIB --> filter2 --> vRIB1 --> filter2 --> vRIB2 --> .. -# -# Now lets break down the various different possible scenarios into more -# detail: -# -# 1. A single physical RIB with no Roto script filtering: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> | pRIB | --> RT -# +------+ -# -# -# -# 2. A single physical RIB with a Roto script filter: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> filter --> RT --> | pRIB | --> RT -# | +------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# -# 3. A physical RIB and a virtual RIB, each with their own Roto script filter: -# -# HTTP API -# ^ | -# | | -# | v -# +------+ -# RT --> filter --> RT --> | pRIB | --> RT -->+ -# | +------+ | -# | | -# v | -# 0..N output streams each | -# emitting values of a single RT | -# v -# +<----------------------------------------+ -# | -# | HTTP API -# | ^ | -# | | | -# | | v -# v +------+ -# +--> filter --> RT --> | vRIB | --> RT -# | +------+ -# | -# v -# 0..N output streams each -# emitting values of a single RT -# -# -# Some notes to be aware of: -# -# - Queries to the HTTP API of a virtual RIB are submitted upstream to the -# physical RIB and the results flow back down the pipeline to the -# requesting virtual RIB and out via its HTTP API. Results are processed -# through each vRIB filter yielding the vRIB modified "view" of the result -# data. -# -# ====================================================================== -# NOTE: Values emitted by output streams of vRIB filters when processing -# HTTP API query results are silently discarded, i.e. values emitted by -# output streams of vRIB filters are only honoured for input data that -# originated to the West of the pRIB, NOT for data that was the result -# of a HTTP API query. -# ====================================================================== -# -# - The input to a physical RIB is usually a Route but can also be a Record -# with a "prefix" key, but only Route values support the notion of being -# "withdrawn". The entire record (all its keys and values) will be added -# to the set of values stored at the prefix in the RIB, with the rib_keys -# fields determining whether a new value is added to the set or replaces -# an existing item in the set. - - - - -### Target: mqtt-out ######################################################### -# -# This target publishes JSON events to an MQTT broker via a TCP connection. -# -# NOTE: The MQTT broker is not part of Rotonda, it is a separate service that -# must be deployed and operated separately to Rotonda. -# -# Tested with the EMQX MQTT broker with both the free public MQTT 5 Broker [1] -# and with the EMQX Docker image [2]. -# -# This target ONLY accepts input data that: -# -# - Was received from a configured upstream source unit. -# - Was emitted by a Roto script output stream. -# - Is of type Record with a "name" field whose value matches the name of -# this instance of the mqtt-out target. -# -# So naming an instance of this unit in a Roto script output stream record is -# not sufficient to have this unit receive it, this unit must still be -# downstream of the producing unit to receive its output. -# -# The JSON event structure produced by this target is a direct serialization -# of the received Roto type as JSON, i.e. a record with a set of key/value -# pairs. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# destination A "host:port" string specifying the host or IP -# address of an MQTT broker to connect to. If the -# ":port" part is omitted the IANA registered MQTT -# port number [3] 1883 will be used. Note: Only -# unencrypted TCP connections are supported, i.e. -# TLS and WS are not supported. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# client_id A unique name to identify the client to the -# (def: "") server in order to hold state about the session. -# If empty the server will use a clean session and -# assign a random name to the client. Servers are -# required to support names upto 23 bytes in length -# but may support more. -# -# qos MQTT quality-of-service setting for determining -# (def: 2) how many times a message can be delivered: -# -# 0 (at most once) -# 1 (at least once) -# 2 (exactly once) -# -# Higher values require more synchronization with -# the broker leading to lower throughput but -# greater reliability/correctness. -# -# queue_size The number of messages that can be buffered for -# (def: 1000) delivery to the MQTT broker. -# -# connect_retry_secs The number of seconds to wait before attempting -# (def: 60) to reconnect to the MQTT broker if the connection -# is lost. -# -# publish_max_secs The number of seconds to wait before timing out -# (def: 5) an attempt to publish a message to the MQTT -# broker. -# -# topic_template A "string" template that will be used to -# (def: "rotonda/{id}") determine the MQTT topic to which events will be -# published. If present, the "{id}" placeholder -# will be replaced by the "topic" value in the -# incoming Record value. When using "{id}" an MQTT -# client that supports MQTT wildcards can still -# receive all events by subscribing to 'rotonda/#' -# for example. -# -# username A "string" username for login to the MQTT broker. -# -# password A "string" password for login to the MQTT broker. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +----------+ -# Record --> | mqtt-out | - - JSON - - > MQTT server -# +----------+ -# -# [1]: https://www.emqx.com/en/mqtt/public-mqtt5-broker -# [2]: https://hub.docker.com/r/emqx/emqx -# [3]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=mqtt - - - - -### Target: null-out ######################################################### -# -# This target discards everything it receives. -# -# Rotonda requires that there always be at least one target. Using this target -# allows you to run Rotonda for testing purposes without any "real" targets, -# or if the only output is via Roto script output stream messages. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# source The upstream unit from which data will be -# received. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +----------+ -# RT --> | null-out | -# +----------+ - - - - -### Target: bmp-fs-out ####################################################### -# -# This target writes raw BMP messages to files on disk, either separated per -# monitored router or merged into a single file. BMP messages can be written -# in one of three different formats: -# -# log, raw, or pcap text -# -# The log format is a limited one line per BMP message plain text log of BMP -# messages received and some limited information about each one. This format -# is intended for gaining a quick insight into the messages being received by -# Rotonda. -# -# The raw format writes the received BMP bytes out as-is, with each BMP -# message byte sequence preceeded by a number indicating how many BMP message -# bytes follow. This format is intended for capturing messages for replay for -# testing purposes later. -# -# THe PCAP text format can be transformed by the separate text2pcap tool, and -# from there can be viewed and analyzed using a tool like WireShark. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# source The upstream unit from which data will be -# received. -# -# path The path to which files will be written. If mode -# is "split" (the default) this setting specifies -# a directory (which must already exist) under -# which one file per router will be created. -# -# format Choose one of: "log", "raw", or "pcaptext". -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# mode Either "merge" or "split". In merged mode all BMP -# (def: "split") messages from all monitored routers are appended -# to the same file. In split mode separate output -# files will be written under an EXISTING directory -# specified by the "path" setting. In "merge" mode -# all BMP messages from all monitored routers will -# be written into a single file file defined by the -# "path" setting. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +------------+ -# BmpMessage --> | bmp-fs-out | - - - - > Writes to disk -# +------------+ - - - - -### Target: bmp-tcp-out ###################################################### -# -# This target writes raw BMP messages over a TCP connection to a specified -# destination IP address and port number, for instance to a second instance of -# Rotonda. -# -# --- Settings --------------------------------------------------------------- -# -# The following MANDATORY settings MUST be specified: -# -# Setting Description -# ======================================================================== -# sources An ["array", "of", "upstream", "unit", "names"] -# from which data will be received. -# -# destination A TCP IP address and port number to proxy raw BMP -# messages to. -# -# The following OPTIONAL settings MAY be specified if desired: -# -# Setting Description -# ======================================================================== -# accept Zero or more "IP address" values defining routers -# (def: []) whose BMP messages will be proxied. If specified, -# ONLY the specified routers will be proxied, all -# others will be able to connect and send messages -# to Rotonda. -# -# reject Zero or more "IP address" values defining routers -# (def: []) whose BMP messages will NOT be proxied. If -# specified and "accept" is NOT specified, this -# setting will permit all other monitored routers -# BMP messages to be proxied. -# -# --- Pipeline interaction --------------------------------------------------- -# -# +-------------+ -# BmpMessage --> | bmp-tcp-out | - - - - > Proxied via TCP/IP -# +-------------+ - - - - -### Sample configuration ##################################################### +# Sample Rotonda configuration file for systemd service log_level = "info" # "error", "warn", "info", "debug" or "trace" log_target = "syslog" # "stderr", "file" or "syslog" log_facility = "daemon" # used if log_target is "syslog" -# Optional path to a directory from which any .roto files present will be -# loaded into Rotonda. Filter names used below refer to blocks in the .roto -# files loaded from this directory. roto_scripts_path = "/etc/rotonda/filters/" http_listen = ["127.0.0.1:8080"] [units.bgp-in] type = "bgp-tcp-in" -listen = "0.0.0.0:179" -my_asn = 64512 # Tip: edit me, or comment this unit out -my_bgp_id = [0, 0, 0, 0] # Tip: edit me, or comment this unit out +listen = "0.0.0.0:179" + +# EDIT THIS: THIS IS AN EXAMPLE VALUE +my_asn = 64512 +# EDIT THIS: THIS IS AN EXAMPLE VALUE +my_bgp_id = [0, 0, 0, 0] + filter_name = "bgp-in-filter" [units.bmp-in] @@ -858,17 +26,6 @@ filter_name = "bmp-in-filter" http_api_path = "/bmp-routers/" tracing_mode = "Off" -# The two RIB units below could have been defined like so: -# -# [units.rib-in-pre] -# type = "rib" -# sources = ["bgp-in", "bmp-in"] -# filter_name = ["rib-in-pre", "rib-in-post"] -# -# But then we don't control the unit naming or HTTP API paths, instead the -# virtual RIB would be named "rib-in-pre-vRIB-0" and its HTTP API path would -# be that of the pRIB with /0/ appended. - [units.rib-in-pre] type = "rib" sources = ["bgp-in", "bmp-in"] @@ -888,14 +45,16 @@ http_api_path = "/rib-in-post/" type = "null-out" sources = ["rib-in-post"] -[targets.bmp-proxy] -type = "bmp-tcp-out" -sources = ["bmp-in"] -destination = "SOME_IP_ADDRESS:SOME_PORT" # Tip: edit me, or comment this target out - +# Commenting out this whole section will disable the mqtt interface [targets.mqtt] type = "mqtt-out" sources = ["bmp-in", "bgp-in", "rib-in-pre", "rib-in-post"] -client_id = "rotonda" # Tip: edit me -topic_template = "{id}" # Tip: edit me -destination = "SOME_IP_ADDRESS:SOME_PORT" # Tip: edit me, or comment this target out + +# EDIT THIS OR COMMENT OUT SECTION +client_id = "rotonda" + +# EDIT THIS OR COMMENT OUT SECTION +topic_template = "{id}" + +# EDIT THIS OR COMMENT OUT SECTION +destination = "SOME_IP_ADDRESS:SOME_PORT" diff --git a/etc/bgp-in-filter.roto b/etc/rotonda/filters/bgp-in-filter.roto similarity index 100% rename from etc/bgp-in-filter.roto rename to etc/rotonda/filters/bgp-in-filter.roto diff --git a/etc/bmp-in-filter.roto b/etc/rotonda/filters/bmp-in-filter.roto similarity index 100% rename from etc/bmp-in-filter.roto rename to etc/rotonda/filters/bmp-in-filter.roto diff --git a/etc/rib-in-post-filter.roto b/etc/rotonda/filters/rib-in-post-filter.roto similarity index 100% rename from etc/rib-in-post-filter.roto rename to etc/rotonda/filters/rib-in-post-filter.roto diff --git a/etc/rib-in-pre-filter.roto b/etc/rotonda/filters/rib-in-pre-filter.roto similarity index 100% rename from etc/rib-in-pre-filter.roto rename to etc/rotonda/filters/rib-in-pre-filter.roto diff --git a/etc/rotonda/rotonda.builtin.conf b/etc/rotonda/rotonda.builtin.conf new file mode 100644 index 0000000..8de8e4c --- /dev/null +++ b/etc/rotonda/rotonda.builtin.conf @@ -0,0 +1,66 @@ +# Rotonda configuration file. + +# For information about this file: https://rotonda.docs.nlnetlabs.nl/en/latest/config/global.html +# This is the configuration file that was used to compile the +log_level = "info" # "error", "warn", "info", "debug" or "trace" +log_target = "stderr" # "stderr", "file" or "syslog" +log_facility = "daemon" # used if log_target is "syslog" +log_file = "./rotonda.log" # used if log_target is "file" + +# Optional path to a directory from which any .roto files present will be +# loaded into Rotonda. Filter names used below refer to blocks in the .roto +# files loaded from this directory. +roto_scripts_path = "/etc/rotonda/filters" + +http_listen = ["127.0.0.1:8080"] + +[units.bgp-in] +type = "bgp-tcp-in" +listen = "0.0.0.0:11179" # TODO: When we can drop privileges make this 179 +my_asn = 64512 # Tip: edit me, or comment this unit out +my_bgp_id = [0, 0, 0, 0] # Tip: edit me, or comment this unit out +filter_name = "bgp-in-filter" + +[units.bmp-in] +type = "bmp-tcp-in" +listen = "0.0.0.0:11019" +filter_name = "bmp-in-filter" +http_api_path = "/bmp-routers/" +tracing_mode = "Off" + +# The two RIB units below could have been defined like so: +# +# [units.rib-in-pre] +# type = "rib" +# sources = ["bgp-in", "bmp-in"] +# filter_name = ["rib-in-pre", "rib-in-post"] +# +# But then we don't control the unit naming or HTTP API paths, instead the +# virtual RIB would be named "rib-in-pre-vRIB-0" and its HTTP API path would +# be that of the pRIB with /0/ appended. + +[units.rib-in-pre] +type = "rib" +sources = ["bgp-in", "bmp-in"] +rib_type = "Physical" +filter_name = "rib-in-pre-filter" +http_api_path = "/rib-in-pre/" + +[units.rib-in-post] +type = "rib" +sources = ["rib-in-pre"] +rib_type = "Virtual" +vrib_upstream = "rib-in-pre" +filter_name = "rib-in-post-filter" +http_api_path = "/rib-in-post/" + +[targets.null] +type = "null-out" +sources = ["rib-in-post"] + +[targets.mqtt] +type = "mqtt-out" +sources = ["bmp-in", "bgp-in", "rib-in-pre", "rib-in-post"] +client_id = "rotonda" # Tip: edit me +topic_template = "{id}" # Tip: edit me +destination = "SOME_IP_ADDRESS:SOME_PORT" # Tip: edit me, or comment this target out diff --git a/etc/rotonda.example.conf b/etc/rotonda/rotonda.example.conf similarity index 82% rename from etc/rotonda.example.conf rename to etc/rotonda/rotonda.example.conf index 9f3029e..a03976d 100644 --- a/etc/rotonda.example.conf +++ b/etc/rotonda/rotonda.example.conf @@ -6,7 +6,7 @@ log_file = "./rotonda.log" # used if log_target is "file" # Optional path to a directory from which any .roto files present will be # loaded into Rotonda. Filter names used below refer to blocks in the .roto # files loaded from this directory. -roto_scripts_path = "etc/" +roto_scripts_path = "/etc/rotonda/filters" http_listen = ["0.0.0.0:8080"] @@ -31,14 +31,6 @@ rib_type = "Physical" filter_name = "rib-in-pre-filter" http_api_path = "/rib-in-pre/" -[units.rib-in-post] -type = "rib" -sources = ["rib-in-pre"] -rib_type = "Virtual" -vrib_upstream = "rib-in-pre" -filter_name = "rib-in-post-filter" -http_api_path = "/rib-in-post/" - [targets.null] type = "null-out" -sources = ["rib-in-post"] +sources = ["rib-in-pre"] diff --git a/src/config.rs b/src/config.rs index 2a0c323..859a763 100644 --- a/src/config.rs +++ b/src/config.rs @@ -132,7 +132,7 @@ impl Config { None => { // No --config command line argument specified, use the embedded MVP config - let bytes = include_bytes!("../etc/rotonda.conf").to_vec(); + let bytes = include_bytes!("../etc/rotonda/rotonda.builtin.conf").to_vec(); // Detect command line arguments designed to override and alter the embedded MVP config file. let mut mvp_overrides = MvpConfig::default(); diff --git a/src/mvp.rs b/src/mvp.rs index 3ae4234..85660bb 100644 --- a/src/mvp.rs +++ b/src/mvp.rs @@ -240,20 +240,20 @@ mod tests { // when the expected roto scripts exist in the mock filesystem let readable_paths: [(PathBuf, String); 4] = [ ( - "etc/bgp-in-filter.roto".into(), - include_str!("../etc/bgp-in-filter.roto").into(), + "/etc/rotonda/filters/bgp-in-filter.roto".into(), + include_str!("../etc/rotonda/filters/bgp-in-filter.roto").into(), ), ( - "etc/bmp-in-filter.roto".into(), - include_str!("../etc/bmp-in-filter.roto").into(), + "/etc/rotonda/filters/bmp-in-filter.roto".into(), + include_str!("../etc/rotonda/filters/bmp-in-filter.roto").into(), ), ( - "etc/rib-in-pre.roto".into(), - include_str!("../etc/rib-in-pre-filter.roto").into(), + "/etc/rotonda/filters/rib-in-pre.roto".into(), + include_str!("../etc/rotonda/filters/rib-in-pre-filter.roto").into(), ), ( - "etc/rib-in-post.roto".into(), - include_str!("../etc/rib-in-post-filter.roto").into(), + "/etc/rotonda/filters/rib-in-post.roto".into(), + include_str!("../etc/rotonda/filters/rib-in-post-filter.roto").into(), ), ]; let mock_io = TheFileIo::new(readable_paths); @@ -268,7 +268,7 @@ mod tests { assert!(!config_source.is_path()); // and the configuration should be correct - assert_eq!(conf.roto_scripts_path, Some("etc/".into())); + assert_eq!(conf.roto_scripts_path, Some("/etc/rotonda/filters".into())); assert_eq!(conf.log.log_target, LogTarget::Stderr); assert_eq!( conf.log.log_level, @@ -308,7 +308,7 @@ mod tests { assert!(!config_source.is_path()); // and the configuration should be correct - assert_eq!(conf.roto_scripts_path, Some("etc/".into())); + assert_eq!(conf.roto_scripts_path, Some("/etc/rotonda/filters".into())); assert_eq!(conf.log.log_target, LogTarget::Stderr); assert_eq!( conf.log.log_level, @@ -347,6 +347,7 @@ mod tests { } #[test] + #[ignore = "BMP proxy currently disabled"] fn mvp_config_with_proxy_destination_cmd_line_arg_should_have_proxy_target( ) { let app = Command::new("test"); @@ -399,6 +400,7 @@ mod tests { } #[test] + #[ignore = "BMP proxy currently disabled"] fn mvp_config_with_proxy_and_mqtt_destination_cmd_line_args_should_have_both_targets( ) { enable_logging("trace");