diff --git a/Cargo.lock b/Cargo.lock index 6ef6caf3..8c66e5ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2446,10 +2446,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] -name = "libyang4-sys" -version = "0.1.1" +name = "libyang5-sys" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21e9b33465f2c872155d44f318b72efac73dd4063690a98e73a31d09552f0f2" +checksum = "8d2af2bd5e6a586aee12e07735a223e232969686ac4ac5b7af639e555ec4a1e1" dependencies = [ "cmake", "pkg-config", @@ -2799,7 +2799,7 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-test", - "yang4", + "yang5", ] [[package]] @@ -3054,12 +3054,13 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "strum 0.28.0", "strum_macros 0.28.0", "tempfile", "tokio", "tracing", "tracing-test", - "yang4", + "yang5", ] [[package]] @@ -6568,14 +6569,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "yang4" -version = "0.1.0" +name = "yang5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f331187c087be4b9e5c823bd48ceec13b3628192278b19ea86b662764fd2d7" +checksum = "1da35149085196bd35c11637c084c0723157c05d97766033cec6d90db535bd19" dependencies = [ "bitflags", "libc", - "libyang4-sys", + "libyang5-sys", "log", "num-derive", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 483e3fcc..5964854f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ anyhow = { version = "1" } serde_with = { version = "3.20" } indexmap = { version = "2.14" } apache-avro = { version = "0.21" } +# TODO: clean-up and consolidate towards schema-registry-client schema_registry_converter = { version = "4.9", default-features = false } rdkafka = { version = "0.39", default-features = false } tracing-test = { version = "0.2" } @@ -102,7 +103,7 @@ sysinfo = { version = "0.39", default-features = false } shadow-rs = { version = "2.0", default-features = false } smallvec = { version = "1.15" } rustc-hash = { version = "2.1" } -yang4 = { version = "0.1", features = ["bundled"] } +yang5 = { version = "0.2", features = ["bundled"] } tempfile = { version = "3.27" } quick-xml = { version = "0.40" } russh = { version = "0.61" } diff --git a/assets/yang/ietf-interfaces/modules/ietf-distributed-notif@2026-04-13.yang b/assets/yang/ietf-interfaces/modules/ietf-distributed-notif@2026-04-13.yang new file mode 100644 index 00000000..70ce99d2 --- /dev/null +++ b/assets/yang/ietf-interfaces/modules/ietf-distributed-notif@2026-04-13.yang @@ -0,0 +1,127 @@ +module ietf-distributed-notif { + yang-version 1.1; + namespace "urn:ietf:params:xml:ns:yang:ietf-distributed-notif"; + prefix dn; + + import ietf-subscribed-notifications { + prefix sn; + reference + "RFC 8639: Subscription to YANG Notifications"; + } + import ietf-yang-push { + prefix yp; + reference + "RFC 8641: Subscription to YANG Notifications for Datastore + Updates"; + } + + organization + "IETF NETCONF (Network Configuration) Working Group"; + contact + "WG Web: + WG List: + + Authors: Guangying Zheng + + Tianran Zhou + + Thomas Graf + + Pierre Francois + + Eric Voit + "; + description + "Defines augmentation for ietf-subscribed-notifications to + enable the distributed publication with single subscription. + + Copyright (c) 2026 IETF Trust and the persons + identified as authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Revised BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (https://trustee.ietf.org/license-info). + + All revisions of IETF and IANA published modules can be found + at the YANG Parameters registry + (https://www.iana.org/assignments/yang-parameters). + + This version of this YANG module is part of RFC XXXX; see + the RFC itself for full legal notices."; + + revision 2026-04-13 { + description + "Initial version"; + reference + "RFC XXXX: Subscription to Notifications in a Distributed + Architecture"; + } + + grouping message-publisher-id { + description + "Provides a reusable message-publisher-id leaf."; + leaf message-publisher-id { + type uint32; + config false; + description + "Identifies the Component software process which publishes + notification messages (e.g., processor 1 on line card 1). This + field is used to notify the receiver which publisher process + published which message. The identifier is locally unique to + the Network Node."; + } + } + + grouping message-publisher-ids { + description + "Provides a reusable message-publisher-id-list leaf-list."; + leaf-list message-publisher-id { + type uint32; + config false; + description + "Identifies the Component software process which publishes + notification messages (e.g., processor 1 on line card 1). This + field is used to notify the receiver which publisher processes + are going to publish. The identifiers are locally unique to + the Network Node."; + } + } + + augment "/sn:subscriptions/sn:subscription" { + description + "This augmentation allows the Message + Publisher ID to be exposed for a subscription."; + uses message-publisher-ids; + } + + augment "/sn:subscription-started" { + description + "This augmentation adds the Message Publisher ID to the + subscription-started subscription change notifications."; + uses message-publisher-ids; + } + + augment "/sn:subscription-modified" { + description + "This augmentation adds the Message Publisher ID to the + subscription-modified subscription change notifications."; + uses message-publisher-ids; + } + + augment "/yp:push-update" { + description + "This augmentation adds the Message Publisher ID in the + push-update notification."; + uses message-publisher-id; + } + + augment "/yp:push-change-update" { + description + "This augmentation adds the Message Publisher ID in the + push-change-update notification."; + uses message-publisher-id; + } +} diff --git a/assets/yang/ietf-interfaces/modules/ietf-interfaces@2014-05-08.yang b/assets/yang/ietf-interfaces/modules/ietf-interfaces@2018-02-20.yang similarity index 52% rename from assets/yang/ietf-interfaces/modules/ietf-interfaces@2014-05-08.yang rename to assets/yang/ietf-interfaces/modules/ietf-interfaces@2018-02-20.yang index ad64425f..f66c205c 100644 --- a/assets/yang/ietf-interfaces/modules/ietf-interfaces@2014-05-08.yang +++ b/assets/yang/ietf-interfaces/modules/ietf-interfaces@2018-02-20.yang @@ -1,5 +1,5 @@ module ietf-interfaces { - + yang-version 1.1; namespace "urn:ietf:params:xml:ns:yang:ietf-interfaces"; prefix if; @@ -8,18 +8,12 @@ module ietf-interfaces { } organization - "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + "IETF NETMOD (Network Modeling) Working Group"; contact - "WG Web: + "WG Web: WG List: - WG Chair: Thomas Nadeau - - - WG Chair: Juergen Schoenwaelder - - Editor: Martin Bjorklund "; @@ -27,7 +21,7 @@ module ietf-interfaces { "This module contains a collection of YANG definitions for managing network interfaces. - Copyright (c) 2014 IETF Trust and the persons identified as + Copyright (c) 2018 IETF Trust and the persons identified as authors of the code. All rights reserved. Redistribution and use in source and binary forms, with or @@ -35,11 +29,18 @@ module ietf-interfaces { to the license terms contained in, the Simplified BSD License set forth in Section 4.c of the IETF Trust's Legal Provisions Relating to IETF Documents - (http://trustee.ietf.org/license-info). + (https://trustee.ietf.org/license-info). - This version of this YANG module is part of RFC 7223; see + This version of this YANG module is part of RFC 8343; see the RFC itself for full legal notices."; + revision 2018-02-20 { + description + "Updated to support NMDA."; + reference + "RFC 8343: A YANG Data Model for Interface Management"; + } + revision 2014-05-08 { description "Initial revision."; @@ -57,16 +58,7 @@ module ietf-interfaces { } description "This type is used by data models that need to reference - configured interfaces."; - } - - typedef interface-state-ref { - type leafref { - path "/if:interfaces-state/if:interface/if:name"; - } - description - "This type is used by data models that need to reference - the operationally present interfaces."; + interfaces."; } /* @@ -95,7 +87,6 @@ module ietf-interfaces { possible to configure an interface whose physical interface hardware is not present on the device."; } - feature if-mib { description "This feature indicates that the device implements @@ -105,29 +96,32 @@ module ietf-interfaces { } /* - * Configuration data nodes + * Data nodes */ container interfaces { description - "Interface configuration parameters."; + "Interface parameters."; list interface { key "name"; description - "The list of configured interfaces on the device. + "The list of interfaces on the device. - The operational state of an interface is available in the - /interfaces-state/interface list. If the configuration of a + The status of an interface is available in this list in the + operational state. If the configuration of a system-controlled interface cannot be used by the system (e.g., the interface hardware present does not match the interface type), then the configuration is not applied to - the system-controlled interface shown in the - /interfaces-state/interface list. If the configuration - of a user-controlled interface cannot be used by the system, - the configured interface is not instantiated in the - /interfaces-state/interface list."; + the system-controlled interface shown in the operational + state. If the configuration of a user-controlled interface + cannot be used by the system, the configured interface is + not instantiated in the operational state. + + System-controlled interfaces created by the system are + always present in this list in the operational state, + whether or not they are configured."; leaf name { type string; @@ -137,18 +131,16 @@ module ietf-interfaces { A device MAY restrict the allowed values for this leaf, possibly depending on the type of the interface. For system-controlled interfaces, this leaf is the - device-specific name of the interface. The 'config false' - list /interfaces-state/interface contains the currently - existing interfaces on the device. + device-specific name of the interface. If a client tries to create configuration for a system-controlled interface that is not present in the - /interfaces-state/interface list, the server MAY reject - the request if the implementation does not support - pre-provisioning of interfaces or if the name refers to - an interface that can never exist in the system. A - NETCONF server MUST reply with an rpc-error with the - error-tag 'invalid-value' in this case. + operational state, the server MAY reject the request if + the implementation does not support pre-provisioning of + interfaces or if the name refers to an interface that can + never exist in the system. A Network Configuration + Protocol (NETCONF) server MUST reply with an rpc-error + with the error-tag 'invalid-value' in this case. If the device supports pre-provisioning of interface configuration, the 'pre-provisioning' feature is @@ -159,7 +151,15 @@ module ietf-interfaces { When a configured user-controlled interface is created by the system, it is instantiated with the same name in the - /interface-state/interface list."; + operational state. + + A server implementation MAY map this leaf to the ifName + MIB object. Such an implementation needs to use some + mechanism to handle the differences in size and characters + allowed between this leaf and ifName. The definition of + such a mechanism is outside the scope of this document."; + reference + "RFC 2863: The Interfaces Group MIB - ifName"; } leaf description { @@ -176,21 +176,7 @@ module ietf-interfaces { Since ifAlias is defined to be stored in non-volatile storage, the MIB implementation MUST map ifAlias to the value of 'description' in the persistently stored - datastore. - - Specifically, if the device supports ':startup', when - ifAlias is read the device MUST return the value of - 'description' in the 'startup' datastore, and when it is - written, it MUST be written to the 'running' and 'startup' - datastores. Note that it is up to the implementation to - - decide whether to modify this single leaf in 'startup' or - perform an implicit copy-config from 'running' to - 'startup'. - - If the device does not support ':startup', ifAlias MUST - be mapped to the 'description' leaf in the 'running' - datastore."; + configuration."; reference "RFC 2863: The Interfaces Group MIB - ifAlias"; } @@ -226,15 +212,12 @@ module ietf-interfaces { interface. Systems that implement the IF-MIB use the value of this - leaf in the 'running' datastore to set + leaf in the intended configuration to set IF-MIB.ifAdminStatus to 'up' or 'down' after an ifEntry has been initialized, as described in RFC 2863. - - - Changes in this leaf in the 'running' datastore are - reflected in ifAdminStatus, but if ifAdminStatus is - changed over SNMP, this leaf is not affected."; + Changes in this leaf in the intended configuration are + reflected in ifAdminStatus."; reference "RFC 2863: The Interfaces Group MIB - ifAdminStatus"; } @@ -244,9 +227,15 @@ module ietf-interfaces { type enumeration { enum enabled { value 1; + description + "The device will generate linkUp/linkDown SNMP + notifications for this interface."; } enum disabled { value 2; + description + "The device will not generate linkUp/linkDown SNMP + notifications for this interface."; } } description @@ -261,34 +250,443 @@ module ietf-interfaces { "RFC 2863: The Interfaces Group MIB - ifLinkUpDownTrapEnable"; } + + leaf admin-status { + if-feature if-mib; + type enumeration { + enum up { + value 1; + description + "Ready to pass packets."; + } + enum down { + value 2; + description + "Not ready to pass packets and not in some test mode."; + } + enum testing { + value 3; + description + "In some test mode."; + } + } + config false; + mandatory true; + description + "The desired state of the interface. + + This leaf has the same read semantics as ifAdminStatus."; + reference + "RFC 2863: The Interfaces Group MIB - ifAdminStatus"; + } + + leaf oper-status { + type enumeration { + enum up { + value 1; + description + "Ready to pass packets."; + } + enum down { + value 2; + + description + "The interface does not pass any packets."; + } + enum testing { + value 3; + description + "In some test mode. No operational packets can + be passed."; + } + enum unknown { + value 4; + description + "Status cannot be determined for some reason."; + } + enum dormant { + value 5; + description + "Waiting for some external event."; + } + enum not-present { + value 6; + description + "Some component (typically hardware) is missing."; + } + enum lower-layer-down { + value 7; + description + "Down due to state of lower-layer interface(s)."; + } + } + config false; + mandatory true; + description + "The current operational state of the interface. + + This leaf has the same semantics as ifOperStatus."; + reference + "RFC 2863: The Interfaces Group MIB - ifOperStatus"; + } + + leaf last-change { + type yang:date-and-time; + config false; + description + "The time the interface entered its current operational + state. If the current state was entered prior to the + last re-initialization of the local network management + subsystem, then this node is not present."; + reference + "RFC 2863: The Interfaces Group MIB - ifLastChange"; + } + + leaf if-index { + if-feature if-mib; + type int32 { + range "1..2147483647"; + } + config false; + mandatory true; + description + "The ifIndex value for the ifEntry represented by this + interface."; + reference + "RFC 2863: The Interfaces Group MIB - ifIndex"; + } + + leaf phys-address { + type yang:phys-address; + config false; + description + "The interface's address at its protocol sub-layer. For + example, for an 802.x interface, this object normally + contains a Media Access Control (MAC) address. The + interface's media-specific modules must define the bit + and byte ordering and the format of the value of this + object. For interfaces that do not have such an address + (e.g., a serial line), this node is not present."; + reference + "RFC 2863: The Interfaces Group MIB - ifPhysAddress"; + } + + leaf-list higher-layer-if { + type interface-ref; + config false; + description + "A list of references to interfaces layered on top of this + interface."; + reference + "RFC 2863: The Interfaces Group MIB - ifStackTable"; + } + + leaf-list lower-layer-if { + type interface-ref; + config false; + + description + "A list of references to interfaces layered underneath this + interface."; + reference + "RFC 2863: The Interfaces Group MIB - ifStackTable"; + } + + leaf speed { + type yang:gauge64; + units "bits/second"; + config false; + description + "An estimate of the interface's current bandwidth in bits + per second. For interfaces that do not vary in + bandwidth or for those where no accurate estimation can + be made, this node should contain the nominal bandwidth. + For interfaces that have no concept of bandwidth, this + node is not present."; + reference + "RFC 2863: The Interfaces Group MIB - + ifSpeed, ifHighSpeed"; + } + + container statistics { + config false; + description + "A collection of interface-related statistics objects."; + + leaf discontinuity-time { + type yang:date-and-time; + mandatory true; + description + "The time on the most recent occasion at which any one or + more of this interface's counters suffered a + discontinuity. If no such discontinuities have occurred + since the last re-initialization of the local management + subsystem, then this node contains the time the local + management subsystem re-initialized itself."; + } + + leaf in-octets { + type yang:counter64; + description + "The total number of octets received on the interface, + including framing characters. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifHCInOctets"; + } + + leaf in-unicast-pkts { + type yang:counter64; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, that were not addressed to a + multicast or broadcast address at this sub-layer. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifHCInUcastPkts"; + } + + leaf in-broadcast-pkts { + type yang:counter64; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, that were addressed to a broadcast + address at this sub-layer. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - + ifHCInBroadcastPkts"; + } + + leaf in-multicast-pkts { + type yang:counter64; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, that were addressed to a multicast + address at this sub-layer. For a MAC-layer protocol, + this includes both Group and Functional addresses. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - + ifHCInMulticastPkts"; + } + + leaf in-discards { + type yang:counter32; + description + "The number of inbound packets that were chosen to be + discarded even though no errors had been detected to + prevent their being deliverable to a higher-layer + protocol. One possible reason for discarding such a + packet could be to free up buffer space. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifInDiscards"; + } + + leaf in-errors { + type yang:counter32; + description + "For packet-oriented interfaces, the number of inbound + packets that contained errors preventing them from being + deliverable to a higher-layer protocol. For character- + oriented or fixed-length interfaces, the number of + inbound transmission units that contained errors + preventing them from being deliverable to a higher-layer + protocol. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifInErrors"; + } + + leaf in-unknown-protos { + type yang:counter32; + + description + "For packet-oriented interfaces, the number of packets + received via the interface that were discarded because + of an unknown or unsupported protocol. For + character-oriented or fixed-length interfaces that + support protocol multiplexing, the number of + transmission units received via the interface that were + discarded because of an unknown or unsupported protocol. + For any interface that does not support protocol + multiplexing, this counter is not present. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifInUnknownProtos"; + } + + leaf out-octets { + type yang:counter64; + description + "The total number of octets transmitted out of the + interface, including framing characters. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifHCOutOctets"; + } + + leaf out-unicast-pkts { + type yang:counter64; + description + "The total number of packets that higher-level protocols + requested be transmitted and that were not addressed + to a multicast or broadcast address at this sub-layer, + including those that were discarded or not sent. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifHCOutUcastPkts"; + } + + leaf out-broadcast-pkts { + type yang:counter64; + description + "The total number of packets that higher-level protocols + requested be transmitted and that were addressed to a + broadcast address at this sub-layer, including those + that were discarded or not sent. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - + ifHCOutBroadcastPkts"; + } + + leaf out-multicast-pkts { + type yang:counter64; + description + "The total number of packets that higher-level protocols + requested be transmitted and that were addressed to a + multicast address at this sub-layer, including those + that were discarded or not sent. For a MAC-layer + protocol, this includes both Group and Functional + addresses. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - + ifHCOutMulticastPkts"; + } + + leaf out-discards { + type yang:counter32; + description + "The number of outbound packets that were chosen to be + discarded even though no errors had been detected to + prevent their being transmitted. One possible reason + for discarding such a packet could be to free up buffer + space. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifOutDiscards"; + } + + leaf out-errors { + type yang:counter32; + description + "For packet-oriented interfaces, the number of outbound + packets that could not be transmitted because of errors. + For character-oriented or fixed-length interfaces, the + number of outbound transmission units that could not be + transmitted because of errors. + + Discontinuities in the value of this counter can occur + at re-initialization of the management system and at + other times as indicated by the value of + 'discontinuity-time'."; + reference + "RFC 2863: The Interfaces Group MIB - ifOutErrors"; + } + } + + } + } + + /* + * Legacy typedefs + */ + + typedef interface-state-ref { + type leafref { + path "/if:interfaces-state/if:interface/if:name"; } + status deprecated; + description + "This type is used by data models that need to reference + the operationally present interfaces."; } /* - * Operational state data nodes + * Legacy operational state data nodes */ container interfaces-state { config false; + status deprecated; description "Data nodes for the operational state of interfaces."; list interface { key "name"; - - - - + status deprecated; description "The list of interfaces on the device. System-controlled interfaces created by the system are - always present in this list, whether they are configured or - not."; + always present in this list, whether or not they are + configured."; leaf name { type string; + status deprecated; description "The name of the interface. @@ -306,6 +704,7 @@ module ietf-interfaces { base interface-type; } mandatory true; + status deprecated; description "The type of the interface."; reference @@ -325,9 +724,6 @@ module ietf-interfaces { description "Not ready to pass packets and not in some test mode."; } - - - enum testing { value 3; description @@ -335,6 +731,7 @@ module ietf-interfaces { } } mandatory true; + status deprecated; description "The desired state of the interface. @@ -383,6 +780,7 @@ module ietf-interfaces { } } mandatory true; + status deprecated; description "The current operational state of the interface. @@ -393,6 +791,7 @@ module ietf-interfaces { leaf last-change { type yang:date-and-time; + status deprecated; description "The time the interface entered its current operational state. If the current state was entered prior to the @@ -408,22 +807,23 @@ module ietf-interfaces { range "1..2147483647"; } mandatory true; + status deprecated; description "The ifIndex value for the ifEntry represented by this interface."; + reference "RFC 2863: The Interfaces Group MIB - ifIndex"; } leaf phys-address { type yang:phys-address; + status deprecated; description "The interface's address at its protocol sub-layer. For example, for an 802.x interface, this object normally contains a Media Access Control (MAC) address. The interface's media-specific modules must define the bit - - and byte ordering and the format of the value of this object. For interfaces that do not have such an address (e.g., a serial line), this node is not present."; @@ -433,6 +833,7 @@ module ietf-interfaces { leaf-list higher-layer-if { type interface-state-ref; + status deprecated; description "A list of references to interfaces layered on top of this interface."; @@ -442,6 +843,7 @@ module ietf-interfaces { leaf-list lower-layer-if { type interface-state-ref; + status deprecated; description "A list of references to interfaces layered underneath this interface."; @@ -452,10 +854,12 @@ module ietf-interfaces { leaf speed { type yang:gauge64; units "bits/second"; + status deprecated; description "An estimate of the interface's current bandwidth in bits per second. For interfaces that do not vary in bandwidth or for those where no accurate estimation can + be made, this node should contain the nominal bandwidth. For interfaces that have no concept of bandwidth, this node is not present."; @@ -464,21 +868,15 @@ module ietf-interfaces { ifSpeed, ifHighSpeed"; } - - - - - - - - container statistics { + status deprecated; description "A collection of interface-related statistics objects."; leaf discontinuity-time { type yang:date-and-time; mandatory true; + status deprecated; description "The time on the most recent occasion at which any one or more of this interface's counters suffered a @@ -490,12 +888,13 @@ module ietf-interfaces { leaf in-octets { type yang:counter64; + status deprecated; description "The total number of octets received on the interface, including framing characters. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -504,31 +903,29 @@ module ietf-interfaces { leaf in-unicast-pkts { type yang:counter64; + status deprecated; description "The number of packets, delivered by this sub-layer to a higher (sub-)layer, that were not addressed to a multicast or broadcast address at this sub-layer. - Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference "RFC 2863: The Interfaces Group MIB - ifHCInUcastPkts"; } - - - leaf in-broadcast-pkts { type yang:counter64; + status deprecated; description "The number of packets, delivered by this sub-layer to a higher (sub-)layer, that were addressed to a broadcast address at this sub-layer. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -538,6 +935,7 @@ module ietf-interfaces { leaf in-multicast-pkts { type yang:counter64; + status deprecated; description "The number of packets, delivered by this sub-layer to a higher (sub-)layer, that were addressed to a multicast @@ -545,7 +943,7 @@ module ietf-interfaces { this includes both Group and Functional addresses. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -555,6 +953,8 @@ module ietf-interfaces { leaf in-discards { type yang:counter32; + status deprecated; + description "The number of inbound packets that were chosen to be discarded even though no errors had been detected to @@ -563,17 +963,16 @@ module ietf-interfaces { packet could be to free up buffer space. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; - - reference "RFC 2863: The Interfaces Group MIB - ifInDiscards"; } leaf in-errors { type yang:counter32; + status deprecated; description "For packet-oriented interfaces, the number of inbound packets that contained errors preventing them from being @@ -584,7 +983,7 @@ module ietf-interfaces { protocol. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -593,6 +992,7 @@ module ietf-interfaces { leaf in-unknown-protos { type yang:counter32; + status deprecated; description "For packet-oriented interfaces, the number of packets received via the interface that were discarded because @@ -603,27 +1003,23 @@ module ietf-interfaces { discarded because of an unknown or unsupported protocol. For any interface that does not support protocol multiplexing, this counter is not present. - Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference "RFC 2863: The Interfaces Group MIB - ifInUnknownProtos"; } - - - - leaf out-octets { type yang:counter64; + status deprecated; description "The total number of octets transmitted out of the interface, including framing characters. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -632,14 +1028,15 @@ module ietf-interfaces { leaf out-unicast-pkts { type yang:counter64; + status deprecated; description "The total number of packets that higher-level protocols - requested be transmitted, and that were not addressed + requested be transmitted and that were not addressed to a multicast or broadcast address at this sub-layer, including those that were discarded or not sent. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -648,14 +1045,16 @@ module ietf-interfaces { leaf out-broadcast-pkts { type yang:counter64; + status deprecated; + description "The total number of packets that higher-level protocols - requested be transmitted, and that were addressed to a + requested be transmitted and that were addressed to a broadcast address at this sub-layer, including those that were discarded or not sent. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -663,19 +1062,19 @@ module ietf-interfaces { ifHCOutBroadcastPkts"; } - leaf out-multicast-pkts { type yang:counter64; + status deprecated; description "The total number of packets that higher-level protocols - requested be transmitted, and that were addressed to a + requested be transmitted and that were addressed to a multicast address at this sub-layer, including those that were discarded or not sent. For a MAC-layer protocol, this includes both Group and Functional addresses. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -685,6 +1084,7 @@ module ietf-interfaces { leaf out-discards { type yang:counter32; + status deprecated; description "The number of outbound packets that were chosen to be discarded even though no errors had been detected to @@ -693,7 +1093,7 @@ module ietf-interfaces { space. Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference @@ -702,6 +1102,7 @@ module ietf-interfaces { leaf out-errors { type yang:counter32; + status deprecated; description "For packet-oriented interfaces, the number of outbound packets that could not be transmitted because of errors. @@ -709,11 +1110,8 @@ module ietf-interfaces { number of outbound transmission units that could not be transmitted because of errors. - - - Discontinuities in the value of this counter can occur - at re-initialization of the management system, and at + at re-initialization of the management system and at other times as indicated by the value of 'discontinuity-time'."; reference diff --git a/assets/yang/ietf-interfaces/modules/ietf-system-capabilities@2022-02-17.yang b/assets/yang/ietf-interfaces/modules/ietf-system-capabilities@2022-02-17.yang index 55f959d8..2a9743ee 100644 --- a/assets/yang/ietf-interfaces/modules/ietf-system-capabilities@2022-02-17.yang +++ b/assets/yang/ietf-interfaces/modules/ietf-system-capabilities@2022-02-17.yang @@ -23,7 +23,7 @@ module ietf-system-capabilities { contact "WG Web: WG List: - + Editor: Balazs Lengyel "; description @@ -31,11 +31,11 @@ module ietf-system-capabilities { capabilities for a server or a publisher. System capabilities may include capabilities of a NETCONF or RESTCONF server or a notification publisher. - + This module does not contain any specific capabilities; it only provides a structure where containers containing the actual capabilities are augmented in. - + Capability values can be specified at the system level, at the datastore level (by selecting all nodes in the datastore), or for specific data nodes of a specific datastore (and their @@ -43,54 +43,54 @@ module ietf-system-capabilities { Capability values specified for a specific datastore or node-set override values specified on the system/publisher level. - + The same grouping MUST be used to define hierarchical capabilities supported both at the system level and at the datastore/data-node level. - + To find a capability value for a specific data node in a specific datastore, the user SHALL: - + 1) search for a datastore-capabilities list entry for the specific datastore. When stating a specific capability, the relative path for any specific capability must be the same under the system-capabilities container and under the per-node-capabilities list. - + 2) If the datastore entry is found within that entry, process all per-node-capabilities entries in the order they appear in the list. The first entry that specifies the specific capability and has a node-selector selecting the specific data node defines the capability value. - + 3) If the capability value is not found above and the specific capability is specified under the system-capabilities container (outside the datastore-capabilities list), this value shall be used. - + 4) If no values are found in the previous steps, the system/publisher is not capable of providing a value. Possible reasons are that it is unknown, the capability is changing for some reason, there is no specified limit, etc. In this case, the system's behavior is unspecified. - + The key words 'MUST', 'MUST NOT', 'REQUIRED', 'SHALL', 'SHALL NOT', 'SHOULD', 'SHOULD NOT', 'RECOMMENDED', 'NOT RECOMMENDED', 'MAY', and 'OPTIONAL' in this document are to be interpreted as described in BCP 14 (RFC 2119) (RFC 8174) when, and only when, they appear in all capitals, as shown here. - + Copyright (c) 2022 IETF Trust and the persons identified as authors of the code. All rights reserved. - + Redistribution and use in source and binary forms, with or without modification, is permitted pursuant to, and subject to the license terms contained in, the Revised BSD License set forth in Section 4.c of the IETF Trust's Legal Provisions Relating to IETF Documents (https://trustee.ietf.org/license-info). - + This version of this YANG module is part of RFC 9196 (https://www.rfc-editor.org/info/rfc9196); see the RFC itself for full legal notices."; @@ -118,13 +118,12 @@ module ietf-system-capabilities { key "datastore"; description "Capabilities values per datastore. - + For non-NMDA servers/publishers, 'config false' data is considered as if it were part of the running datastore."; leaf datastore { type leafref { - path - "/yanglib:yang-library/yanglib:datastore/yanglib:name"; + path "/yanglib:yang-library/yanglib:datastore/yanglib:name"; } description "The datastore for which capabilities are defined. @@ -137,11 +136,11 @@ module ietf-system-capabilities { "Each list entry specifies capabilities for the selected data nodes. The same capabilities apply to the data nodes in the subtree below the selected nodes. - + The system SHALL order the entries according to their precedence. The order of the entries MUST NOT change unless the underlying capabilities also change. - + Note that the longest patch matching can be achieved by ordering more specific matches before less specific ones."; @@ -160,10 +159,10 @@ module ietf-system-capabilities { "RFC 8341: Network Configuration Access Control Model"; } } - /* - * "Augmentation point for datastore- or data-node-level - * capabilities." - */ + /* + * "Augmentation point for datastore- or data-node-level + * capabilities." + */ } } } diff --git a/assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-08-29.yang b/assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-12-07.yang similarity index 78% rename from assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-08-29.yang rename to assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-12-07.yang index 731f654e..b714a42f 100644 --- a/assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-08-29.yang +++ b/assets/yang/ietf-interfaces/modules/ietf-yang-push-revision@2025-12-07.yang @@ -84,7 +84,7 @@ module ietf-yang-push-revision { described in BCP 14 (RFC 2119) (RFC 8174) when, and only when, they appear in all capitals, as shown here."; - revision 2025-08-29 { + revision 2025-12-07 { description "First revision"; reference @@ -92,13 +92,6 @@ module ietf-yang-push-revision { Subscription"; } - feature yang-push-revision-supported { - description - "This feature indicates the YANG Subscription Notifications - supports specifying the list of modules, revisions and - version in the YANG subscription."; - } - // Identities identity revision-unsupported { base sn:establish-subscription-error; @@ -113,7 +106,7 @@ module ietf-yang-push-revision { base sn:establish-subscription-error; base sn:modify-subscription-error; description - "Version not supported. This failure can be due to + "Specific version not supported. This failure can be due to subscribing to a specific version not supported by the publisher."; } @@ -122,14 +115,13 @@ module ietf-yang-push-revision { base sn:establish-subscription-error; base sn:modify-subscription-error; description - "The combination of revision and the version are + "The combination of revision and version are incompatible. This failure happens when the revision and the version are both specified in the RPC and the YANG - module supported by the publisher does not support one of the - revision or the version."; + module supported by the publisher does not support one of them."; } - grouping yang-push-module-version-config { + grouping yang-push-module-version-subs { description "This grouping combines the module name, the revision and version leaves. This grouping is to be used for @@ -139,21 +131,30 @@ module ietf-yang-push-revision { description "This references the YANG module name."; } - leaf revision { - type rev:revision-date; - description - "This references the YANG module revision to be sent in the - subscription."; - } - leaf version { - type ysver:version; + choice revision-version { description - "This references the YANG module semantic version to be sent + "Specifies the revision or version of the YANG module sent in the subscription."; - } + case revision { + leaf revision { + type rev:revision-date; + description + "This references the YANG module revision to be sent in the + subscription."; + } + } + case version { + leaf version { + type ysver:version; + description + "This references the YANG module semantic version to be sent + in the subscription."; + } + } + } } - grouping yang-push-module-version { + grouping yang-push-module-version-notif { description "This grouping combines the module name, the revision and version leaves. This grouping is to be used for @@ -183,9 +184,9 @@ module ietf-yang-push-revision { } } - grouping yang-push-module-version-list { + grouping yang-push-module-version-notif-list { description - "This grouping defines a list of yang-push-module-version + "This grouping defines a list of yang-push-module-version-notif grouping."; list module-version { key "name"; @@ -193,7 +194,7 @@ module ietf-yang-push-revision { description "List of yang-push-module-version grouping. The revision is not configurable."; - uses ypr:yang-push-module-version; + uses ypr:yang-push-module-version-notif; } leaf yang-library-content-id { type leafref { @@ -206,78 +207,75 @@ module ietf-yang-push-revision { } } - grouping yang-push-module-version-config-list { + grouping yang-push-module-version-subs-list { description - "This grouping defines a list of yang-push-module-version-config + "This grouping defines a list of yang-push-module-version-subs grouping."; - list module-version-config { + list module-version-subs { key "name"; description - "List of yang-push-module-version-config grouping. The + "List of yang-push-module-version-subs grouping. The revision is configurable."; - uses ypr:yang-push-module-version-config; + uses ypr:yang-push-module-version-subs; } } // Subscription parameters augment "/sn:establish-subscription/sn:input" { - if-feature "yang-push-revision-supported"; description "Augment the establish-subscription RPC from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } augment "/sn:modify-subscription/sn:input" { - if-feature "yang-push-revision-supported"; description "Augment the modify-subscription RPC from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } // Subscription notifications augment "/sn:subscription-started" { - if-feature "yang-push-revision-supported"; description "Augment the subscription-started notification from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-list grouping."; - uses ypr:yang-push-module-version-list; + yang-push-module-version-notif-list grouping."; + uses ypr:yang-push-module-version-notif-list; } augment "/sn:subscription-modified" { - if-feature "yang-push-revision-supported"; description "Augment the subscription-modified notification from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-list grouping."; - uses ypr:yang-push-module-version-list; + yang-push-module-version-notif-list grouping."; + uses ypr:yang-push-module-version-notif-list; } // Subscription container augment "/sn:subscriptions/sn:subscription" { - if-feature "yang-push-revision-supported"; description "Augment the subscriptions RPC container from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } - // Event capabilities + // Subscription capabilities augment "/sysc:system-capabilities/notc:subscription-capabilities" { description "Add system level capabilities"; leaf yang-push-module-revision-supported { type boolean; + default true; description "Specifies whether the publisher supports exporting revision and version in YANG-Push subscription state change - notifications."; + notifications. If set to true, publisher supports. If set to + false, publisher does not support."; reference "RFC XXXX: Support of Versioning in YANG Notifications Subscription"; } } -} \ No newline at end of file +} diff --git a/assets/yang/ietf-interfaces/modules/ietf-yp-observation@2026-05-11.yang b/assets/yang/ietf-interfaces/modules/ietf-yp-observation@2026-05-11.yang new file mode 100644 index 00000000..8a245621 --- /dev/null +++ b/assets/yang/ietf-interfaces/modules/ietf-yp-observation@2026-05-11.yang @@ -0,0 +1,138 @@ +module ietf-yp-observation { + yang-version 1.1; + namespace "urn:ietf:params:xml:ns:yang:ietf-yp-observation"; + prefix iypo; + + import ietf-yang-types { + prefix yang; + reference + "RFC 9911: Common YANG Data Types"; + } + import ietf-yang-push { + prefix yp; + reference + "RFC 8641: Subscription to YANG Notifications for Datastore + Updates"; + } + import ietf-system-capabilities { + prefix sysc; + reference + "RFC 9196: YANG Modules Describing Capabilities for + Systems and Datastore Update Notifications"; + } + import ietf-notification-capabilities { + prefix notc; + reference + "RFC 9196: YANG Modules Describing Capabilities for + Systems and Datastore Update Notifications"; + } + + organization + "IETF NETCONF (Network Configuration) Working Group"; + contact + "WG Web: + WG List: + + Authors: Thomas Graf + + Benoit Claise + + Alex Huang Feng + "; + description + "Defines YANG-Push event notification header with the observation + time in streaming update notifications. + + Copyright (c) 2026 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Revised BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (https://trustee.ietf.org/license-info). + + All revisions of IETF and IANA published modules can be found + at the YANG Parameters registry group + (https://www.iana.org/assignments/yang-parameters). + + This version of this YANG module is part of RFC XXXX; see + the RFC itself for full legal notices."; + + revision 2026-05-11 { + description + "Initial version."; + reference + "RFC XXXX: Extensible YANG Model for YANG-Push Notifications"; + } + + grouping yang-push-observation { + description + "This grouping adds the observation timestamp for the + observed metrics."; + leaf timestamp { + type yang:date-and-time; + description + "This is the time when the metrics were observed."; + } + leaf point-in-time { + type enumeration { + enum current-accounting { + description + "For periodic subscriptions, the point-in-time + where the metrics are being polled and observed."; + } + enum initial-state { + description + "For 'on-change sync on start' subscriptions, the + initial point in time when the subscription was + established and the state was observed."; + } + enum state-changed { + description + "For 'on-change sync on start' subscriptions, the + point in time when the state change was observed after + the subscription was established."; + } + } + description + "This describes at which point in time the metrics were + observed."; + } + } + + // Event notifications + augment "/yp:push-update" { + description + "This augmentation adds the observation timestamp of the + accounted metrics in the push-update notification."; + uses iypo:yang-push-observation; + } + + augment "/yp:push-change-update" { + description + "This augmentation adds the observation timestamp of the + event in the push-change-update notification."; + uses iypo:yang-push-observation; + } + + // Event capabilities + augment "/sysc:system-capabilities" + + "/notc:subscription-capabilities" { + description + "Add YANG-Push notification capabilities to system-level + capability container."; + leaf yang-push-observation-time-supported { + type boolean; + default "false"; + description + "Specifies whether the publisher supports exporting + observation-timestamp and point-in-time in notifications. + If set to true, publisher supports. If set to false, + the observation-timestamp is not supported."; + reference + "RFC XXXX: Extensible YANG Model for YANG-Push Notifications"; + } + } +} diff --git a/assets/yang/ietf-interfaces/subscriptions-info.json b/assets/yang/ietf-interfaces/subscriptions-info.json index b7778881..24ba4642 100644 --- a/assets/yang/ietf-interfaces/subscriptions-info.json +++ b/assets/yang/ietf-interfaces/subscriptions-info.json @@ -1,7 +1,6 @@ [ { - "collector": "127.0.0.1:10000", - "peer": "0.0.0.0:830", + "peer_ip": "0.0.0.0", "id": 1, "target": { "ietf-yang-push:datastore": "ietf-datastores:operational", @@ -26,4 +25,4 @@ ], "content_id": "test-content-id-1" } -] +] \ No newline at end of file diff --git a/assets/yang/ietf-interfaces/yang-lib.xml b/assets/yang/ietf-interfaces/yang-lib.xml index ab19d8d8..8bd3d20e 100644 --- a/assets/yang/ietf-interfaces/yang-lib.xml +++ b/assets/yang/ietf-interfaces/yang-lib.xml @@ -1,190 +1,259 @@ - + complete + + + + + + + + - ietf-interfaces - 2014-05-08 - urn:ietf:params:xml:ns:yang:ietf-interfaces - arbitrary-names - pre-provisioning - if-mib - ietf-ip - ietf-network-instance - + ietf-yp-notification + 2025-02-24 + urn:ietf:params:xml:ns:yang:ietf-yp-notification - + - ietf-yang-types - 2013-07-15 - urn:ietf:params:xml:ns:yang:ietf-yang-types + ietf-system-capabilities + 2022-02-17 + urn:ietf:params:xml:ns:yang:ietf-system-capabilities - ietf-inet-types - 2021-02-22 - urn:ietf:params:xml:ns:yang:ietf-inet-types + ietf-notification-capabilities + 2022-02-17 + urn:ietf:params:xml:ns:yang:ietf-notification-capabilities + + - ietf-platform-manifest - 2025-02-21 - urn:ietf:params:xml:ns:yang:ietf-platform-manifest + ietf-yang-push + 2019-09-09 + urn:ietf:params:xml:ns:yang:ietf-yang-push + + on-change + - ietf-yang-structure-ext - 2020-06-17 - urn:ietf:params:xml:ns:yang:ietf-yang-structure-ext + ietf-yang-push-revision + 2025-12-07 + urn:ietf:params:xml:ns:yang:ietf-yang-push-revision - + - ietf-subscribed-notifications - 2019-09-09 - urn:ietf:params:xml:ns:yang:ietf-subscribed-notifications - - encode-json - encode-xml - configured - subtree - xpath + ietf-yp-observation + 2026-05-11 + urn:ietf:params:xml:ns:yang:ietf-yp-observation + + - ietf-yang-push - 2019-09-09 - urn:ietf:params:xml:ns:yang:ietf-yang-push - - on-change + ietf-subscribed-notifications + 2019-09-09 + urn:ietf:params:xml:ns:yang:ietf-subscribed-notifications + + encode-json + encode-xml + configured + subtree + xpath + - ietf-datastores - 2018-02-14 - urn:ietf:params:xml:ns:yang:ietf-datastores + ietf-distributed-notif + 2026-04-13 + urn:ietf:params:xml:ns:yang:ietf-distributed-notif + - ietf-yang-revisions - 2024-06-04 - urn:ietf:params:xml:ns:yang:ietf-yang-revisions + ietf-netconf-acm + 2018-02-14 + urn:ietf:params:xml:ns:yang:ietf-netconf-acm - ietf-yang-semver - 2024-07-02 - urn:ietf:params:xml:ns:yang:ietf-yang-semver + ietf-restconf + 2017-01-26 + urn:ietf:params:xml:ns:yang:ietf-restconf + + + + + ietf-platform-manifest + 2025-02-21 + urn:ietf:params:xml:ns:yang:ietf-platform-manifest - ietf-yang-library - 2019-01-04 - urn:ietf:params:xml:ns:yang:ietf-yang-library + ietf-yang-library + 2019-01-04 + urn:ietf:params:xml:ns:yang:ietf-yang-library + + + + + + + + + + ietf-udp-notif-transport + 2025-02-14 + urn:ietf:params:xml:ns:yang:ietf-udp-notif-transport + + + + ietf-subscribed-notif-receivers + 2024-02-01 + urn:ietf:params:xml:ns:yang:ietf-subscribed-notif-receivers - - ietf-netconf-acm - 2018-02-14 - urn:ietf:params:xml:ns:yang:ietf-netconf-acm + ietf-udp-client + 2025-02-24 + urn:ietf:params:xml:ns:yang:ietf-udp-client - ietf-network-instance - 2019-01-21 - urn:ietf:params:xml:ns:yang:ietf-network-instance + ietf-tls-client + 2024-10-10 + urn:ietf:params:xml:ns:yang:ietf-tls-client + - ietf-restconf - 2017-01-26 - urn:ietf:params:xml:ns:yang:ietf-restconf + ietf-crypto-types + 2024-10-10 + urn:ietf:params:xml:ns:yang:ietf-crypto-types - - ietf-yang-patch - 2017-02-22 - urn:ietf:params:xml:ns:yang:ietf-yang-patch + ietf-truststore + 2024-10-10 + urn:ietf:params:xml:ns:yang:ietf-truststore - - ietf-ip - 2018-02-22 - urn:ietf:params:xml:ns:yang:ietf-ip + ietf-keystore + 2024-10-10 + urn:ietf:params:xml:ns:yang:ietf-keystore - ietf-yang-schema-mount - 2019-01-14 - urn:ietf:params:xml:ns:yang:ietf-yang-schema-mount + ietf-tls-common + 2024-10-10 + urn:ietf:params:xml:ns:yang:ietf-tls-common - + - ietf-yp-notification - 2025-02-24 - urn:ietf:params:xml:ns:yang:ietf-yp-notification + iana-tls-cipher-suite-algs + 2024-10-16 + urn:ietf:params:xml:ns:yang:iana-tls-cipher-suite-algs - + + + + + + + - ietf-system-capabilities - 2022-02-17 - urn:ietf:params:xml:ns:yang:ietf-system-capabilities + ietf-yang-types + 2013-07-15 + urn:ietf:params:xml:ns:yang:ietf-yang-types - ietf-notification-capabilities - 2022-02-17 - urn:ietf:params:xml:ns:yang:ietf-notification-capabilities + ietf-inet-types + 2021-02-22 + urn:ietf:params:xml:ns:yang:ietf-inet-types - + - ietf-udp-notif-transport - 2025-02-14 - urn:ietf:params:xml:ns:yang:ietf-udp-notif-transport + ietf-datastores + 2018-02-14 + urn:ietf:params:xml:ns:yang:ietf-datastores - + - ietf-subscribed-notif-receivers - 2024-02-01 - urn:ietf:params:xml:ns:yang:ietf-subscribed-notif-receivers + ietf-yang-patch + 2017-02-22 + urn:ietf:params:xml:ns:yang:ietf-yang-patch + - ietf-udp-client - 2025-02-24 - urn:ietf:params:xml:ns:yang:ietf-udp-client + ietf-yang-revisions + 2024-06-04 + urn:ietf:params:xml:ns:yang:ietf-yang-revisions + - ietf-tls-client - 2024-10-10 - urn:ietf:params:xml:ns:yang:ietf-tls-client + ietf-yang-semver + 2024-07-02 + urn:ietf:params:xml:ns:yang:ietf-yang-semver - + - iana-if-type - 2014-05-08 - urn:ietf:params:xml:ns:yang:iana-if-type + ietf-yang-structure-ext + 2020-06-17 + urn:ietf:params:xml:ns:yang:ietf-yang-structure-ext - + - ietf-crypto-types - 2024-10-10 - urn:ietf:params:xml:ns:yang:ietf-crypto-types + ietf-yang-schema-mount + 2019-01-14 + urn:ietf:params:xml:ns:yang:ietf-yang-schema-mount + + + + + + + + - ietf-truststore - 2024-10-10 - urn:ietf:params:xml:ns:yang:ietf-truststore + ietf-interfaces + 2018-02-20 + urn:ietf:params:xml:ns:yang:ietf-interfaces + arbitrary-names + pre-provisioning + if-mib + + ietf-ip + + ietf-network-instance + - ietf-keystore - 2024-10-10 - urn:ietf:params:xml:ns:yang:ietf-keystore + ietf-ip + 2018-02-22 + urn:ietf:params:xml:ns:yang:ietf-ip + - ietf-tls-common - 2024-10-10 - urn:ietf:params:xml:ns:yang:ietf-tls-common + ietf-network-instance + 2019-01-21 + urn:ietf:params:xml:ns:yang:ietf-network-instance - + - iana-tls-cipher-suite-algs - 2024-10-16 - urn:ietf:params:xml:ns:yang:iana-tls-cipher-suite-algs + iana-if-type + 2014-05-08 + urn:ietf:params:xml:ns:yang:iana-if-type - - + + + complete complete - -ietf-telemetry-message-full + + yp-envelope-ietf-interface -34 - + 34 + \ No newline at end of file diff --git a/assets/yang/ietf-telemetry-message/modules/ietf-distributed-notif@2026-04-13.yang b/assets/yang/ietf-telemetry-message/modules/ietf-distributed-notif@2026-04-13.yang new file mode 100644 index 00000000..70ce99d2 --- /dev/null +++ b/assets/yang/ietf-telemetry-message/modules/ietf-distributed-notif@2026-04-13.yang @@ -0,0 +1,127 @@ +module ietf-distributed-notif { + yang-version 1.1; + namespace "urn:ietf:params:xml:ns:yang:ietf-distributed-notif"; + prefix dn; + + import ietf-subscribed-notifications { + prefix sn; + reference + "RFC 8639: Subscription to YANG Notifications"; + } + import ietf-yang-push { + prefix yp; + reference + "RFC 8641: Subscription to YANG Notifications for Datastore + Updates"; + } + + organization + "IETF NETCONF (Network Configuration) Working Group"; + contact + "WG Web: + WG List: + + Authors: Guangying Zheng + + Tianran Zhou + + Thomas Graf + + Pierre Francois + + Eric Voit + "; + description + "Defines augmentation for ietf-subscribed-notifications to + enable the distributed publication with single subscription. + + Copyright (c) 2026 IETF Trust and the persons + identified as authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Revised BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (https://trustee.ietf.org/license-info). + + All revisions of IETF and IANA published modules can be found + at the YANG Parameters registry + (https://www.iana.org/assignments/yang-parameters). + + This version of this YANG module is part of RFC XXXX; see + the RFC itself for full legal notices."; + + revision 2026-04-13 { + description + "Initial version"; + reference + "RFC XXXX: Subscription to Notifications in a Distributed + Architecture"; + } + + grouping message-publisher-id { + description + "Provides a reusable message-publisher-id leaf."; + leaf message-publisher-id { + type uint32; + config false; + description + "Identifies the Component software process which publishes + notification messages (e.g., processor 1 on line card 1). This + field is used to notify the receiver which publisher process + published which message. The identifier is locally unique to + the Network Node."; + } + } + + grouping message-publisher-ids { + description + "Provides a reusable message-publisher-id-list leaf-list."; + leaf-list message-publisher-id { + type uint32; + config false; + description + "Identifies the Component software process which publishes + notification messages (e.g., processor 1 on line card 1). This + field is used to notify the receiver which publisher processes + are going to publish. The identifiers are locally unique to + the Network Node."; + } + } + + augment "/sn:subscriptions/sn:subscription" { + description + "This augmentation allows the Message + Publisher ID to be exposed for a subscription."; + uses message-publisher-ids; + } + + augment "/sn:subscription-started" { + description + "This augmentation adds the Message Publisher ID to the + subscription-started subscription change notifications."; + uses message-publisher-ids; + } + + augment "/sn:subscription-modified" { + description + "This augmentation adds the Message Publisher ID to the + subscription-modified subscription change notifications."; + uses message-publisher-ids; + } + + augment "/yp:push-update" { + description + "This augmentation adds the Message Publisher ID in the + push-update notification."; + uses message-publisher-id; + } + + augment "/yp:push-change-update" { + description + "This augmentation adds the Message Publisher ID in the + push-change-update notification."; + uses message-publisher-id; + } +} diff --git a/assets/yang/ietf-telemetry-message/modules/ietf-system-capabilities@2022-02-17.yang b/assets/yang/ietf-telemetry-message/modules/ietf-system-capabilities@2022-02-17.yang index 55f959d8..2a9743ee 100644 --- a/assets/yang/ietf-telemetry-message/modules/ietf-system-capabilities@2022-02-17.yang +++ b/assets/yang/ietf-telemetry-message/modules/ietf-system-capabilities@2022-02-17.yang @@ -23,7 +23,7 @@ module ietf-system-capabilities { contact "WG Web: WG List: - + Editor: Balazs Lengyel "; description @@ -31,11 +31,11 @@ module ietf-system-capabilities { capabilities for a server or a publisher. System capabilities may include capabilities of a NETCONF or RESTCONF server or a notification publisher. - + This module does not contain any specific capabilities; it only provides a structure where containers containing the actual capabilities are augmented in. - + Capability values can be specified at the system level, at the datastore level (by selecting all nodes in the datastore), or for specific data nodes of a specific datastore (and their @@ -43,54 +43,54 @@ module ietf-system-capabilities { Capability values specified for a specific datastore or node-set override values specified on the system/publisher level. - + The same grouping MUST be used to define hierarchical capabilities supported both at the system level and at the datastore/data-node level. - + To find a capability value for a specific data node in a specific datastore, the user SHALL: - + 1) search for a datastore-capabilities list entry for the specific datastore. When stating a specific capability, the relative path for any specific capability must be the same under the system-capabilities container and under the per-node-capabilities list. - + 2) If the datastore entry is found within that entry, process all per-node-capabilities entries in the order they appear in the list. The first entry that specifies the specific capability and has a node-selector selecting the specific data node defines the capability value. - + 3) If the capability value is not found above and the specific capability is specified under the system-capabilities container (outside the datastore-capabilities list), this value shall be used. - + 4) If no values are found in the previous steps, the system/publisher is not capable of providing a value. Possible reasons are that it is unknown, the capability is changing for some reason, there is no specified limit, etc. In this case, the system's behavior is unspecified. - + The key words 'MUST', 'MUST NOT', 'REQUIRED', 'SHALL', 'SHALL NOT', 'SHOULD', 'SHOULD NOT', 'RECOMMENDED', 'NOT RECOMMENDED', 'MAY', and 'OPTIONAL' in this document are to be interpreted as described in BCP 14 (RFC 2119) (RFC 8174) when, and only when, they appear in all capitals, as shown here. - + Copyright (c) 2022 IETF Trust and the persons identified as authors of the code. All rights reserved. - + Redistribution and use in source and binary forms, with or without modification, is permitted pursuant to, and subject to the license terms contained in, the Revised BSD License set forth in Section 4.c of the IETF Trust's Legal Provisions Relating to IETF Documents (https://trustee.ietf.org/license-info). - + This version of this YANG module is part of RFC 9196 (https://www.rfc-editor.org/info/rfc9196); see the RFC itself for full legal notices."; @@ -118,13 +118,12 @@ module ietf-system-capabilities { key "datastore"; description "Capabilities values per datastore. - + For non-NMDA servers/publishers, 'config false' data is considered as if it were part of the running datastore."; leaf datastore { type leafref { - path - "/yanglib:yang-library/yanglib:datastore/yanglib:name"; + path "/yanglib:yang-library/yanglib:datastore/yanglib:name"; } description "The datastore for which capabilities are defined. @@ -137,11 +136,11 @@ module ietf-system-capabilities { "Each list entry specifies capabilities for the selected data nodes. The same capabilities apply to the data nodes in the subtree below the selected nodes. - + The system SHALL order the entries according to their precedence. The order of the entries MUST NOT change unless the underlying capabilities also change. - + Note that the longest patch matching can be achieved by ordering more specific matches before less specific ones."; @@ -160,10 +159,10 @@ module ietf-system-capabilities { "RFC 8341: Network Configuration Access Control Model"; } } - /* - * "Augmentation point for datastore- or data-node-level - * capabilities." - */ + /* + * "Augmentation point for datastore- or data-node-level + * capabilities." + */ } } } diff --git a/assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-08-29.yang b/assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-12-07.yang similarity index 78% rename from assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-08-29.yang rename to assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-12-07.yang index 731f654e..b714a42f 100644 --- a/assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-08-29.yang +++ b/assets/yang/ietf-telemetry-message/modules/ietf-yang-push-revision@2025-12-07.yang @@ -84,7 +84,7 @@ module ietf-yang-push-revision { described in BCP 14 (RFC 2119) (RFC 8174) when, and only when, they appear in all capitals, as shown here."; - revision 2025-08-29 { + revision 2025-12-07 { description "First revision"; reference @@ -92,13 +92,6 @@ module ietf-yang-push-revision { Subscription"; } - feature yang-push-revision-supported { - description - "This feature indicates the YANG Subscription Notifications - supports specifying the list of modules, revisions and - version in the YANG subscription."; - } - // Identities identity revision-unsupported { base sn:establish-subscription-error; @@ -113,7 +106,7 @@ module ietf-yang-push-revision { base sn:establish-subscription-error; base sn:modify-subscription-error; description - "Version not supported. This failure can be due to + "Specific version not supported. This failure can be due to subscribing to a specific version not supported by the publisher."; } @@ -122,14 +115,13 @@ module ietf-yang-push-revision { base sn:establish-subscription-error; base sn:modify-subscription-error; description - "The combination of revision and the version are + "The combination of revision and version are incompatible. This failure happens when the revision and the version are both specified in the RPC and the YANG - module supported by the publisher does not support one of the - revision or the version."; + module supported by the publisher does not support one of them."; } - grouping yang-push-module-version-config { + grouping yang-push-module-version-subs { description "This grouping combines the module name, the revision and version leaves. This grouping is to be used for @@ -139,21 +131,30 @@ module ietf-yang-push-revision { description "This references the YANG module name."; } - leaf revision { - type rev:revision-date; - description - "This references the YANG module revision to be sent in the - subscription."; - } - leaf version { - type ysver:version; + choice revision-version { description - "This references the YANG module semantic version to be sent + "Specifies the revision or version of the YANG module sent in the subscription."; - } + case revision { + leaf revision { + type rev:revision-date; + description + "This references the YANG module revision to be sent in the + subscription."; + } + } + case version { + leaf version { + type ysver:version; + description + "This references the YANG module semantic version to be sent + in the subscription."; + } + } + } } - grouping yang-push-module-version { + grouping yang-push-module-version-notif { description "This grouping combines the module name, the revision and version leaves. This grouping is to be used for @@ -183,9 +184,9 @@ module ietf-yang-push-revision { } } - grouping yang-push-module-version-list { + grouping yang-push-module-version-notif-list { description - "This grouping defines a list of yang-push-module-version + "This grouping defines a list of yang-push-module-version-notif grouping."; list module-version { key "name"; @@ -193,7 +194,7 @@ module ietf-yang-push-revision { description "List of yang-push-module-version grouping. The revision is not configurable."; - uses ypr:yang-push-module-version; + uses ypr:yang-push-module-version-notif; } leaf yang-library-content-id { type leafref { @@ -206,78 +207,75 @@ module ietf-yang-push-revision { } } - grouping yang-push-module-version-config-list { + grouping yang-push-module-version-subs-list { description - "This grouping defines a list of yang-push-module-version-config + "This grouping defines a list of yang-push-module-version-subs grouping."; - list module-version-config { + list module-version-subs { key "name"; description - "List of yang-push-module-version-config grouping. The + "List of yang-push-module-version-subs grouping. The revision is configurable."; - uses ypr:yang-push-module-version-config; + uses ypr:yang-push-module-version-subs; } } // Subscription parameters augment "/sn:establish-subscription/sn:input" { - if-feature "yang-push-revision-supported"; description "Augment the establish-subscription RPC from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } augment "/sn:modify-subscription/sn:input" { - if-feature "yang-push-revision-supported"; description "Augment the modify-subscription RPC from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } // Subscription notifications augment "/sn:subscription-started" { - if-feature "yang-push-revision-supported"; description "Augment the subscription-started notification from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-list grouping."; - uses ypr:yang-push-module-version-list; + yang-push-module-version-notif-list grouping."; + uses ypr:yang-push-module-version-notif-list; } augment "/sn:subscription-modified" { - if-feature "yang-push-revision-supported"; description "Augment the subscription-modified notification from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-list grouping."; - uses ypr:yang-push-module-version-list; + yang-push-module-version-notif-list grouping."; + uses ypr:yang-push-module-version-notif-list; } // Subscription container augment "/sn:subscriptions/sn:subscription" { - if-feature "yang-push-revision-supported"; description "Augment the subscriptions RPC container from the ietf-subscribed-notifications YANG module with the - yang-push-module-version-config-list grouping."; - uses ypr:yang-push-module-version-config-list; + yang-push-module-version-subs-list grouping."; + uses ypr:yang-push-module-version-subs-list; } - // Event capabilities + // Subscription capabilities augment "/sysc:system-capabilities/notc:subscription-capabilities" { description "Add system level capabilities"; leaf yang-push-module-revision-supported { type boolean; + default true; description "Specifies whether the publisher supports exporting revision and version in YANG-Push subscription state change - notifications."; + notifications. If set to true, publisher supports. If set to + false, publisher does not support."; reference "RFC XXXX: Support of Versioning in YANG Notifications Subscription"; } } -} \ No newline at end of file +} diff --git a/assets/yang/ietf-telemetry-message/modules/ietf-yp-observation@2026-05-11.yang b/assets/yang/ietf-telemetry-message/modules/ietf-yp-observation@2026-05-11.yang new file mode 100644 index 00000000..8a245621 --- /dev/null +++ b/assets/yang/ietf-telemetry-message/modules/ietf-yp-observation@2026-05-11.yang @@ -0,0 +1,138 @@ +module ietf-yp-observation { + yang-version 1.1; + namespace "urn:ietf:params:xml:ns:yang:ietf-yp-observation"; + prefix iypo; + + import ietf-yang-types { + prefix yang; + reference + "RFC 9911: Common YANG Data Types"; + } + import ietf-yang-push { + prefix yp; + reference + "RFC 8641: Subscription to YANG Notifications for Datastore + Updates"; + } + import ietf-system-capabilities { + prefix sysc; + reference + "RFC 9196: YANG Modules Describing Capabilities for + Systems and Datastore Update Notifications"; + } + import ietf-notification-capabilities { + prefix notc; + reference + "RFC 9196: YANG Modules Describing Capabilities for + Systems and Datastore Update Notifications"; + } + + organization + "IETF NETCONF (Network Configuration) Working Group"; + contact + "WG Web: + WG List: + + Authors: Thomas Graf + + Benoit Claise + + Alex Huang Feng + "; + description + "Defines YANG-Push event notification header with the observation + time in streaming update notifications. + + Copyright (c) 2026 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Revised BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (https://trustee.ietf.org/license-info). + + All revisions of IETF and IANA published modules can be found + at the YANG Parameters registry group + (https://www.iana.org/assignments/yang-parameters). + + This version of this YANG module is part of RFC XXXX; see + the RFC itself for full legal notices."; + + revision 2026-05-11 { + description + "Initial version."; + reference + "RFC XXXX: Extensible YANG Model for YANG-Push Notifications"; + } + + grouping yang-push-observation { + description + "This grouping adds the observation timestamp for the + observed metrics."; + leaf timestamp { + type yang:date-and-time; + description + "This is the time when the metrics were observed."; + } + leaf point-in-time { + type enumeration { + enum current-accounting { + description + "For periodic subscriptions, the point-in-time + where the metrics are being polled and observed."; + } + enum initial-state { + description + "For 'on-change sync on start' subscriptions, the + initial point in time when the subscription was + established and the state was observed."; + } + enum state-changed { + description + "For 'on-change sync on start' subscriptions, the + point in time when the state change was observed after + the subscription was established."; + } + } + description + "This describes at which point in time the metrics were + observed."; + } + } + + // Event notifications + augment "/yp:push-update" { + description + "This augmentation adds the observation timestamp of the + accounted metrics in the push-update notification."; + uses iypo:yang-push-observation; + } + + augment "/yp:push-change-update" { + description + "This augmentation adds the observation timestamp of the + event in the push-change-update notification."; + uses iypo:yang-push-observation; + } + + // Event capabilities + augment "/sysc:system-capabilities" + + "/notc:subscription-capabilities" { + description + "Add YANG-Push notification capabilities to system-level + capability container."; + leaf yang-push-observation-time-supported { + type boolean; + default "false"; + description + "Specifies whether the publisher supports exporting + observation-timestamp and point-in-time in notifications. + If set to true, publisher supports. If set to false, + the observation-timestamp is not supported."; + reference + "RFC XXXX: Extensible YANG Model for YANG-Push Notifications"; + } + } +} diff --git a/assets/yang/ietf-telemetry-message/subscriptions-info.json b/assets/yang/ietf-telemetry-message/subscriptions-info.json index d1081e60..7c0f6503 100644 --- a/assets/yang/ietf-telemetry-message/subscriptions-info.json +++ b/assets/yang/ietf-telemetry-message/subscriptions-info.json @@ -1,6 +1,6 @@ [ { - "peer": "0.0.0.0:830", + "peer_ip": "0.0.0.0", "id": 1, "content_id": "ietf-telemetry-message", "target": { @@ -12,4 +12,4 @@ "ietf-yang-push-telemetry-message" ] } -] +] \ No newline at end of file diff --git a/assets/yang/ietf-telemetry-message/yang-lib-full.xml b/assets/yang/ietf-telemetry-message/yang-lib-full.xml index aa31e4f2..1c90cb0a 100644 --- a/assets/yang/ietf-telemetry-message/yang-lib-full.xml +++ b/assets/yang/ietf-telemetry-message/yang-lib-full.xml @@ -1,7 +1,15 @@ complete - + + + + + + + + + ietf-telemetry-message 2025-10-19 @@ -9,44 +17,41 @@ network-node-manifest data-collection-manifest + ietf-yang-push-telemetry-message 2025-10-19 urn:ietf:params:xml:ns:yang:ietf-yang-push-telemetry-message - + + + + + + + + - ietf-yang-types - 2013-07-15 - urn:ietf:params:xml:ns:yang:ietf-yang-types - - - ietf-inet-types - 2021-02-22 - urn:ietf:params:xml:ns:yang:ietf-inet-types - - - ietf-platform-manifest - 2025-02-21 - urn:ietf:params:xml:ns:yang:ietf-platform-manifest + ietf-yp-notification + 2025-02-24 + urn:ietf:params:xml:ns:yang:ietf-yp-notification + - ietf-yang-structure-ext - 2020-06-17 - urn:ietf:params:xml:ns:yang:ietf-yang-structure-ext + ietf-system-capabilities + 2022-02-17 + urn:ietf:params:xml:ns:yang:ietf-system-capabilities - - ietf-subscribed-notifications - 2019-09-09 - urn:ietf:params:xml:ns:yang:ietf-subscribed-notifications - - encode-json - encode-xml - configured - subtree - xpath + ietf-notification-capabilities + 2022-02-17 + urn:ietf:params:xml:ns:yang:ietf-notification-capabilities + + ietf-yang-push 2019-09-09 @@ -54,89 +59,79 @@ on-change + - ietf-datastores - 2018-02-14 - urn:ietf:params:xml:ns:yang:ietf-datastores - - - ietf-yang-revisions - 2024-06-04 - urn:ietf:params:xml:ns:yang:ietf-yang-revisions + ietf-yang-push-revision + 2025-12-07 + urn:ietf:params:xml:ns:yang:ietf-yang-push-revision + - ietf-yang-semver - 2024-07-02 - urn:ietf:params:xml:ns:yang:ietf-yang-semver + ietf-yp-observation + 2026-05-11 + urn:ietf:params:xml:ns:yang:ietf-yp-observation - + + - ietf-yang-library - 2019-01-04 - urn:ietf:params:xml:ns:yang:ietf-yang-library + ietf-subscribed-notifications + 2019-09-09 + urn:ietf:params:xml:ns:yang:ietf-subscribed-notifications + + encode-json + encode-xml + configured + subtree + xpath - + - ietf-interfaces - 2018-02-20 - urn:ietf:params:xml:ns:yang:ietf-interfaces + ietf-distributed-notif + 2026-04-13 + urn:ietf:params:xml:ns:yang:ietf-distributed-notif + ietf-netconf-acm 2018-02-14 urn:ietf:params:xml:ns:yang:ietf-netconf-acm - - ietf-network-instance - 2019-01-21 - urn:ietf:params:xml:ns:yang:ietf-network-instance - ietf-restconf 2017-01-26 urn:ietf:params:xml:ns:yang:ietf-restconf - - - ietf-yang-patch - 2017-02-22 - urn:ietf:params:xml:ns:yang:ietf-yang-patch - - - - ietf-ip - 2018-02-22 - urn:ietf:params:xml:ns:yang:ietf-ip - - - ietf-yang-schema-mount - 2019-01-14 - urn:ietf:params:xml:ns:yang:ietf-yang-schema-mount - - + + - ietf-yp-notification - 2025-02-24 - urn:ietf:params:xml:ns:yang:ietf-yp-notification - - - - ietf-system-capabilities - 2022-02-17 - urn:ietf:params:xml:ns:yang:ietf-system-capabilities + ietf-platform-manifest + 2025-02-21 + urn:ietf:params:xml:ns:yang:ietf-platform-manifest + - ietf-notification-capabilities - 2022-02-17 - urn:ietf:params:xml:ns:yang:ietf-notification-capabilities + ietf-yang-library + 2019-01-04 + urn:ietf:params:xml:ns:yang:ietf-yang-library - + + + + + + + ietf-udp-notif-transport 2025-02-14 urn:ietf:params:xml:ns:yang:ietf-udp-notif-transport - + ietf-subscribed-notif-receivers 2024-02-01 @@ -152,13 +147,7 @@ 2024-10-10 urn:ietf:params:xml:ns:yang:ietf-tls-client - - - iana-if-type - 2014-05-08 - urn:ietf:params:xml:ns:yang:iana-if-type - - + ietf-crypto-types 2024-10-10 @@ -179,12 +168,112 @@ 2024-10-10 urn:ietf:params:xml:ns:yang:ietf-tls-common - + iana-tls-cipher-suite-algs 2024-10-16 urn:ietf:params:xml:ns:yang:iana-tls-cipher-suite-algs + + + + + + + + + + ietf-yang-types + 2013-07-15 + urn:ietf:params:xml:ns:yang:ietf-yang-types + + + + ietf-inet-types + 2021-02-22 + urn:ietf:params:xml:ns:yang:ietf-inet-types + + + + ietf-datastores + 2018-02-14 + urn:ietf:params:xml:ns:yang:ietf-datastores + + + + ietf-yang-patch + 2017-02-22 + urn:ietf:params:xml:ns:yang:ietf-yang-patch + + + + ietf-yang-revisions + 2024-06-04 + urn:ietf:params:xml:ns:yang:ietf-yang-revisions + + + + ietf-yang-semver + 2024-07-02 + urn:ietf:params:xml:ns:yang:ietf-yang-semver + + + + ietf-yang-structure-ext + 2020-06-17 + urn:ietf:params:xml:ns:yang:ietf-yang-structure-ext + + + + ietf-yang-schema-mount + 2019-01-14 + urn:ietf:params:xml:ns:yang:ietf-yang-schema-mount + + + + + + + + + + + ietf-interfaces + 2018-02-20 + urn:ietf:params:xml:ns:yang:ietf-interfaces + + ietf-ip + + ietf-network-instance + + + + ietf-ip + 2018-02-22 + urn:ietf:params:xml:ns:yang:ietf-ip + + + + ietf-network-instance + 2019-01-21 + urn:ietf:params:xml:ns:yang:ietf-network-instance + + + + iana-if-type + 2014-05-08 + urn:ietf:params:xml:ns:yang:iana-if-type + + complete @@ -194,4 +283,4 @@ 34 - + \ No newline at end of file diff --git a/assets/yang/ietf-telemetry-message/yang-lib-min.xml b/assets/yang/ietf-telemetry-message/yang-lib-min.xml index 8e6548ac..0a0a6394 100644 --- a/assets/yang/ietf-telemetry-message/yang-lib-min.xml +++ b/assets/yang/ietf-telemetry-message/yang-lib-min.xml @@ -44,4 +44,4 @@ 34 - + \ No newline at end of file diff --git a/crates/collector/Cargo.toml b/crates/collector/Cargo.toml index 86b3cc26..a88c0ac4 100644 --- a/crates/collector/Cargo.toml +++ b/crates/collector/Cargo.toml @@ -125,7 +125,7 @@ chrono = { workspace = true, default-features = false, features = [ "clock", ] } tempfile = { workspace = true } -yang4 = { workspace = true} +yang5 = { workspace = true } tracing-test = { workspace = true } [features] diff --git a/crates/collector/examples/kafka-yang-consumer.rs b/crates/collector/examples/kafka-yang-consumer.rs index 961d2d91..8436589a 100644 --- a/crates/collector/examples/kafka-yang-consumer.rs +++ b/crates/collector/examples/kafka-yang-consumer.rs @@ -1,3 +1,4 @@ +// Copyright (C) 2026-present The NetCalyx Authors. // Copyright (C) 2026-present The NetGauze Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,11 +46,11 @@ use schema_registry_client::rest::schema_registry_client::{Client, SchemaRegistr use serde_json::json; use shadow_rs::shadow; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use tokio::signal; use tracing::{debug, error, info, trace, warn}; -use yang4::context::Context; -use yang4::data::{DataFormat, DataParserFlags, DataValidationFlags}; +use yang5::context::Context; +use yang5::data::{DataFormat, DataParserFlags, DataValidationFlags}; shadow!(build); @@ -131,7 +132,6 @@ struct ValidationStats { failed: usize, context_errors: usize, no_schema: usize, - no_tm_schema: usize, } /// Cache for YANG contexts indexed by schema ID @@ -206,7 +206,7 @@ impl YangContextCache { &yang_lib_ref.yang_library_path(), DataFormat::XML, &search_dir.as_path(), - yang4::context::ContextFlags::empty(), + yang5::context::ContextFlags::empty(), )?; info!( @@ -803,7 +803,6 @@ fn log_statistics(stats: &ValidationStats, total_count: usize) { info!(" Failed: {}", stats.failed); info!(" Context errors: {}", stats.context_errors); info!(" No schema-id (skipped): {}", stats.no_schema); - info!(" No ietf-tm schema (skipped): {}", stats.no_tm_schema); info!(""); let validated_total = stats.passed + stats.failed; @@ -846,12 +845,8 @@ async fn main() -> Result<()> { let sr_client = SchemaRegistryClient::new(sr_config); // Create placeholder subscription info (reused for all schemas) - let subscription_info = SubscriptionInfo::new_empty( - SocketAddr::new(IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 0), - None, - SocketAddr::new(IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 0), - 0, - ); + let subscription_info = + SubscriptionInfo::new_empty(IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 0); // Create YANG context cache let mut yang_ctx_cache = YangContextCache::new(); @@ -971,64 +966,36 @@ async fn main() -> Result<()> { Ok(yang_ctx) => { debug!("Using YANG context for schema ID: {}", schema_id); - // Get Telemetry Message module - let tm_module = - yang_ctx.get_module_implemented("ietf-telemetry-message"); - - if tm_module.is_none() { - warn!( - "Message validation SKIPPED [partition: {}, offset: {}, key: {}]: \ - ietf-telemetry-message schema not found", - borrowed_message.partition(), - borrowed_message.offset(), - String::from_utf8_lossy(key) - ); - validation_stats.no_tm_schema += 1; - } else { - // Extract ietf-telemetry-message extension instance - let tm_ext = tm_module.as_ref().and_then(|m| m.extensions().next()); - - // Validate message payload - let validation_result = match &tm_ext { - Some(ext) => yang4::data::DataTree::parse_ext_string( - ext, - payload, - DataFormat::JSON, - DataParserFlags::STRICT, - DataValidationFlags::PRESENT, - ), - // Support legacy ietf-telemetry-message without YANG structure - None => yang4::data::DataTree::parse_string( - yang_ctx, - payload, - DataFormat::JSON, - DataParserFlags::STRICT, - DataValidationFlags::PRESENT, - ), - }; - - match validation_result { - Ok(_) => { - debug!( - "Message validation PASSED [partition: {}, offset: {}, key: {}, schema_id: {}]", - borrowed_message.partition(), - borrowed_message.offset(), - String::from_utf8_lossy(key), - schema_id - ); - validation_stats.passed += 1; - } - Err(err) => { - error!( - "Message validation FAILED [partition: {}, offset: {}, key: {}, schema_id: {}]: {}", - borrowed_message.partition(), - borrowed_message.offset(), - String::from_utf8_lossy(key), - schema_id, - err - ); - validation_stats.failed += 1; - } + // Validate message payload against YANG schemas. + let validation_result = yang5::data::DataTree::parse_string( + yang_ctx, + payload, + DataFormat::JSON, + DataParserFlags::STRICT | DataParserFlags::ANYDATA_STRICT, + DataValidationFlags::PRESENT, + ); + + match validation_result { + Ok(_) => { + debug!( + "Message validation PASSED [partition: {}, offset: {}, key: {}, schema_id: {}]", + borrowed_message.partition(), + borrowed_message.offset(), + String::from_utf8_lossy(key), + schema_id + ); + validation_stats.passed += 1; + } + Err(err) => { + error!( + "Message validation FAILED [partition: {}, offset: {}, key: {}, schema_id: {}]: {}", + borrowed_message.partition(), + borrowed_message.offset(), + String::from_utf8_lossy(key), + schema_id, + err + ); + validation_stats.failed += 1; } } } diff --git a/crates/collector/src/lib.rs b/crates/collector/src/lib.rs index f4ae77b2..50412619 100644 --- a/crates/collector/src/lib.rs +++ b/crates/collector/src/lib.rs @@ -1,3 +1,4 @@ +// Copyright (C) 2026-present The NetCalyx Authors. // Copyright (C) 2024-present The NetGauze Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +25,7 @@ use crate::publishers::http::{HttpPublisherActorHandle, Message}; use crate::publishers::kafka_avro::KafkaAvroPublisherActorHandle; use crate::publishers::kafka_json::KafkaJsonPublisherActorHandle; use crate::publishers::kafka_yang::KafkaYangPublisherActorHandle; -use crate::yang_push::enrichment::YangPushEnrichmentActorHandle; +use crate::yang_push::enrichment::{EnrichedNotification, YangPushEnrichmentActorHandle}; use futures_util::StreamExt; use futures_util::stream::FuturesUnordered; @@ -36,11 +37,9 @@ use netcalyx_flow_service::flow_supervisor::FlowCollectorsSupervisorActorHandle; use netcalyx_udp_notif_pkt::raw::MediaType; use netcalyx_udp_notif_service::UdpNotifRequest; use netcalyx_udp_notif_service::supervisor::UdpNotifSupervisorHandle; -use netcalyx_yang_push::ContentId; use netcalyx_yang_push::cache::actor::CacheActorHandle; use netcalyx_yang_push::cache::fetcher::{NetconfYangLibraryFetcher, RetryConfig}; -use netcalyx_yang_push::cache::storage::SubscriptionInfo; -use netcalyx_yang_push::model::telemetry::{Manifest, TelemetryMessageWrapper}; +use netcalyx_yang_push::model::telemetry::Manifest; use netcalyx_yang_push::validation::ValidationActorHandle; use shadow_rs::shadow; use std::net::IpAddr; @@ -926,10 +925,10 @@ fn serialize_udp_notif( } fn serialize_telemetry_json( - input: (Option, SubscriptionInfo, TelemetryMessageWrapper), + input: EnrichedNotification, _writer_id: String, ) -> Result<(Option, serde_json::Value), UdpNotifSerializationError> { - let tmw = input.2; + let tmw = input.message; let ip = tmw.message().telemetry_message_metadata().export_address(); let value = serde_json::to_value(tmw)?; let key = serde_json::Value::String(ip.to_string()); @@ -1054,6 +1053,7 @@ mod tests { use super::*; use bytes::Bytes; use netcalyx_udp_notif_pkt::raw::UdpNotifPacket; + use netcalyx_udp_notif_service::SessionInfo; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -1070,7 +1070,10 @@ mod tests { Bytes::from(&[0xffu8, 0xffu8][..]), ); - let request = Arc::new(UdpNotifRequest::new(collector, None, peer, pkt)); + let request = Arc::new(UdpNotifRequest::new( + SessionInfo::new(collector, None, peer), + pkt, + )); let serialized = serialize_udp_notif(request.clone(), writer_id.clone()); assert!(matches!( serialized, @@ -1112,12 +1115,13 @@ mod tests { } ); let request_invalid = Arc::new(UdpNotifRequest::new( - collector, - None, - peer, + SessionInfo::new(collector, None, peer), pkt_invalid_json, )); - let request_good = Arc::new(UdpNotifRequest::new(collector, None, peer, pkt)); + let request_good = Arc::new(UdpNotifRequest::new( + SessionInfo::new(collector, None, peer), + pkt, + )); let result_invalid = serialize_udp_notif(request_invalid, writer_id.clone()); let serialized = serialize_udp_notif(request_good, writer_id.clone()).expect("failed to serialize json"); @@ -1168,12 +1172,13 @@ mod tests { ); let request_invalid = Arc::new(UdpNotifRequest::new( - collector, - None, - peer, + SessionInfo::new(collector, None, peer), pkt_invalid_utf8, )); - let request_good = Arc::new(UdpNotifRequest::new(collector, None, peer, pkt)); + let request_good = Arc::new(UdpNotifRequest::new( + SessionInfo::new(collector, None, peer), + pkt, + )); let result_invalid = serialize_udp_notif(request_invalid, writer_id.clone()); let serialized = serialize_udp_notif(request_good, writer_id.clone()).expect("failed to serialize json"); @@ -1226,8 +1231,14 @@ mod tests { } ); - let request_invalid = Arc::new(UdpNotifRequest::new(collector, None, peer, pkt_invalid)); - let request_good = Arc::new(UdpNotifRequest::new(collector, None, peer, pkt)); + let request_invalid = Arc::new(UdpNotifRequest::new( + SessionInfo::new(collector, None, peer), + pkt_invalid, + )); + let request_good = Arc::new(UdpNotifRequest::new( + SessionInfo::new(collector, None, peer), + pkt, + )); let result_invalid = serialize_udp_notif(request_invalid, writer_id.clone()); let serialized = serialize_udp_notif(request_good, writer_id.clone()).expect("failed to serialize json"); diff --git a/crates/collector/src/publishers/kafka_yang.rs b/crates/collector/src/publishers/kafka_yang.rs index 21a86c90..394a2177 100644 --- a/crates/collector/src/publishers/kafka_yang.rs +++ b/crates/collector/src/publishers/kafka_yang.rs @@ -1,3 +1,4 @@ +// Copyright (C) 2026-present The NetCalyx Authors. // Copyright (C) 2025-present The NetGauze Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -430,7 +431,7 @@ where return if let Some(default_schema_id) = self.default_schema_id { if let Some(subscription_info) = subscription_info { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -449,7 +450,7 @@ where } else { if let Some(subscription_info) = subscription_info { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -470,7 +471,7 @@ where if let Some(&schema_id) = self.schema_id_cache.get(id) { if let Some(subscription_info) = subscription_info { trace!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), diff --git a/crates/collector/src/yang_push/config.rs b/crates/collector/src/yang_push/config.rs index 87c79e6f..39970096 100644 --- a/crates/collector/src/yang_push/config.rs +++ b/crates/collector/src/yang_push/config.rs @@ -1,3 +1,4 @@ +// Copyright (C) 2026-present The NetCalyx Authors. // Copyright (C) 2025-present The NetGauze Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,9 +22,9 @@ //! JSON format. use crate::publishers::kafka_yang::YangConverter; +use crate::yang_push::enrichment::EnrichedNotification; use netcalyx_yang_push::ContentId; use netcalyx_yang_push::cache::storage::{SubscriptionInfo, YangLibraryReference}; -use netcalyx_yang_push::model::telemetry::TelemetryMessageWrapper; use serde::{Deserialize, Serialize}; #[derive(Debug, strum_macros::Display)] @@ -81,12 +82,7 @@ impl TelemetryYangConverter { } } -impl - YangConverter< - (Option, SubscriptionInfo, TelemetryMessageWrapper), - TelemetryYangConverterError, - > for TelemetryYangConverter -{ +impl YangConverter for TelemetryYangConverter { fn subject_prefix(&self) -> Option<&str> { self.subject_prefix.as_deref() } @@ -103,38 +99,27 @@ impl self.extension_yang_lib_ref.as_ref() } - fn content_id( - &self, - input: &(Option, SubscriptionInfo, TelemetryMessageWrapper), - ) -> Option { - input.0.clone() + fn content_id(&self, input: &EnrichedNotification) -> Option { + input.cached_content_id.clone() } - fn get_key( - &self, - input: &(Option, SubscriptionInfo, TelemetryMessageWrapper), - ) -> Option { - let (_, subscription_info, _) = input; - let ip = subscription_info.peer().ip(); + fn get_key(&self, input: &EnrichedNotification) -> Option { + let ip = input.subscription_info.peer_ip(); Some(serde_json::Value::String(ip.to_string())) } fn serialize_json( &self, - input: (Option, SubscriptionInfo, TelemetryMessageWrapper), + input: EnrichedNotification, ) -> Result, TelemetryYangConverterError> { - let telemetry_message_wrapper = input.2; - serde_json::to_vec(&telemetry_message_wrapper).map_err(Into::into) + serde_json::to_vec(&input.message).map_err(Into::into) } - fn subscription_info( - &self, - input: &(Option, SubscriptionInfo, TelemetryMessageWrapper), - ) -> Option { - if input.1.is_empty() { + fn subscription_info(&self, input: &EnrichedNotification) -> Option { + if input.subscription_info.is_empty() { None } else { - Some(input.1.clone()) + Some(input.subscription_info.clone()) } } } @@ -145,14 +130,13 @@ mod tests { use chrono::TimeZone; use netcalyx_netconf_proto::yang_push::identities::{Encoding, Transport}; use netcalyx_netconf_proto::yang_push::subscription::YangPushModuleVersion; + use netcalyx_udp_notif_service::SessionInfo; use netcalyx_yang_push::model::telemetry::*; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; fn create_test_subscription_info(ip: IpAddr) -> SubscriptionInfo { SubscriptionInfo::new( - SocketAddr::from(([127, 0, 0, 1], 10000)), - None, - SocketAddr::new(ip, 8080), + ip, 1, netcalyx_udp_notif_pkt::notification::Target::new_datastore( "ietf-datastores:operational".to_string(), @@ -223,7 +207,16 @@ mod tests { let sub_info = create_test_subscription_info(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); let msg = create_test_telemetry_message_wrapper(); - let input = (content_id.clone(), sub_info, msg); + let input = EnrichedNotification { + cached_content_id: content_id.clone(), + subscription_info: sub_info, + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 0)), + None, + SocketAddr::from(([127, 0, 0, 1], 0)), + ), + message: msg, + }; assert_eq!(converter.content_id(&input), content_id); } @@ -234,7 +227,16 @@ mod tests { let sub_info = create_test_subscription_info(ip); let msg = create_test_telemetry_message_wrapper(); - let input = (None, sub_info, msg); + let input = EnrichedNotification { + cached_content_id: None, + subscription_info: sub_info, + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 0)), + None, + SocketAddr::from(([127, 0, 0, 1], 0)), + ), + message: msg, + }; let key = converter.get_key(&input).unwrap(); assert_eq!(key, serde_json::Value::String("192.168.1.1".to_string())); } @@ -246,7 +248,16 @@ mod tests { let sub_info = create_test_subscription_info(ip); let msg = create_test_telemetry_message_wrapper(); - let input = (None, sub_info, msg); + let input = EnrichedNotification { + cached_content_id: None, + subscription_info: sub_info, + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 0)), + None, + SocketAddr::from(([127, 0, 0, 1], 0)), + ), + message: msg, + }; let key = converter.get_key(&input).unwrap(); assert_eq!(key, serde_json::Value::String("2001:db8::1".to_string())); } @@ -259,7 +270,16 @@ mod tests { let expected = serde_json::to_value(&msg).unwrap(); // Call serialize_json to serialize into bytes - let input = (None, sub_info, msg); + let input = EnrichedNotification { + cached_content_id: None, + subscription_info: sub_info, + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 0)), + None, + SocketAddr::from(([127, 0, 0, 1], 0)), + ), + message: msg, + }; let result = converter.serialize_json(input); assert!(result.is_ok()); diff --git a/crates/collector/src/yang_push/enrichment.rs b/crates/collector/src/yang_push/enrichment.rs index 4586984f..cb6fffae 100644 --- a/crates/collector/src/yang_push/enrichment.rs +++ b/crates/collector/src/yang_push/enrichment.rs @@ -36,12 +36,13 @@ use crate::yang_push::{ }; use chrono::Utc; use netcalyx_udp_notif_pkt::decoded::{UdpNotifPacketDecoded, UdpNotifPayload}; -use netcalyx_udp_notif_service::OTL_UDP_NOTIF_PUBLISHER_ID_KEY; +use netcalyx_udp_notif_service::{OTL_UDP_NOTIF_PUBLISHER_ID_KEY, SessionInfo}; use netcalyx_yang_push::cache::storage::SubscriptionInfo; use netcalyx_yang_push::model::telemetry::{ EventType, Label, Manifest, NetworkOperatorMetadata, SessionProtocol, TelemetryMessage, TelemetryMessageMetadata, TelemetryMessageWrapper, YangPushSubscriptionMetadata, }; +use netcalyx_yang_push::validation::ValidatedNotification; use netcalyx_yang_push::{ ContentId, OTL_YANG_PUSH_CACHED_CONTENT_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_ROUTER_CONTENT_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_TARGET_KEY, @@ -150,15 +151,31 @@ impl YangPushEnrichmentStats { } } +/// The output of the enrichment stage: a `TelemetryMessage` assembled from a +/// validated UDP-Notif packet, together with the subscription identity and +/// transport session that produced it. +/// +/// - `cached_content_id`: the YANG schema fingerprint used for validation, or +/// `None` when the packet was forwarded unvalidated. +/// - `subscription_info`: subscription identity (peer IP, target, modules…). +/// - `session`: transport session context (collector, interface, full peer +/// `SocketAddr`) — available for logging, routing, and Kafka key selection. +/// - `message`: the enriched telemetry message ready for publishing. +#[derive(Debug)] +pub struct EnrichedNotification { + pub cached_content_id: Option, + pub subscription_info: SubscriptionInfo, + pub session: SessionInfo, + pub message: TelemetryMessageWrapper, +} + /// Actor responsible for enriching YANG-Push notifications. /// Sends enriched TelemetryMessage objects. struct YangPushEnrichmentActor { cmd_rx: mpsc::Receiver, enrichment_rx: async_channel::Receiver, - validated_rx: - async_channel::Receiver<(Option, SubscriptionInfo, UdpNotifPacketDecoded)>, - enriched_tx: - async_channel::Sender<(Option, SubscriptionInfo, TelemetryMessageWrapper)>, + validated_rx: async_channel::Receiver, + enriched_tx: async_channel::Sender, labels: HashMap>, manifest: Manifest, stats: YangPushEnrichmentStats, @@ -168,16 +185,8 @@ impl YangPushEnrichmentActor { fn new( cmd_rx: mpsc::Receiver, enrichment_rx: async_channel::Receiver, - validated_rx: async_channel::Receiver<( - Option, - SubscriptionInfo, - UdpNotifPacketDecoded, - )>, - enriched_tx: async_channel::Sender<( - Option, - SubscriptionInfo, - TelemetryMessageWrapper, - )>, + validated_rx: async_channel::Receiver, + enriched_tx: async_channel::Sender, manifest: Manifest, stats: YangPushEnrichmentStats, ) -> Self { @@ -340,12 +349,13 @@ impl YangPushEnrichmentActor { &mut self, content_id: Option<&ContentId>, subscription_info: &SubscriptionInfo, + session: &SessionInfo, decoded_packet: &UdpNotifPacketDecoded, ) -> Result { if decoded_packet.notification_type().is_none() { return Err(YangPushEnrichmentActorError::NotificationWithoutContent); } - let peer = subscription_info.peer(); + let peer_ip = subscription_info.peer_ip(); let message_id = decoded_packet.message_id(); let publisher_id = decoded_packet.publisher_id(); let notification_type = decoded_packet @@ -355,7 +365,7 @@ impl YangPushEnrichmentActor { let labels: Option> = self .labels - .get(&peer.ip()) + .get(&peer_ip) .map(|l_map| l_map.values().cloned().map(|wl| wl.label).collect()); // Match on the wrapper and process the notification content @@ -378,17 +388,17 @@ impl YangPushEnrichmentActor { EventType::Log, None, // we don't set sequence numbers for now SessionProtocol::YangPush, // only option at the moment - peer.ip(), - Some(peer.port()), - None, - None, + peer_ip, + Some(session.peer().port()), + Some(session.collector().ip()), + Some(session.collector().port()), subscription_metadata, ); // Re-serialize the UDP-Notif payload into JSON let json_payload = serde_json::to_value(decoded_packet.payload()).map_err(|err| { error!( - peer=%peer, + peer_ip=%peer_ip, message_id, publisher_id, subscription_id=subscription_info.id(), @@ -444,17 +454,13 @@ impl YangPushEnrichmentActor { msg = self.validated_rx.recv() => { match msg { Ok(msg) => { - let (content_id, subscription_info, pkt) = msg; - let peer = subscription_info.peer(); + let ValidatedNotification { cached_content_id: content_id, subscription_info, session, packet: pkt } = msg; + let peer_ip = subscription_info.peer_ip(); let publisher_id = pkt.publisher_id(); let peer_tags = [ opentelemetry::KeyValue::new( "network.peer.address", - format!("{}", peer.ip()), - ), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(peer.port().into()), + format!("{peer_ip}"), ), opentelemetry::KeyValue::new( OTL_UDP_NOTIF_PUBLISHER_ID_KEY, @@ -480,9 +486,14 @@ impl YangPushEnrichmentActor { self.stats.received_messages.add(1, &peer_tags); // Process the payload and send the enriched TelemetryMessage - match self.process_decoded_udp_notif_packet(content_id.as_ref(), &subscription_info, &pkt) { + match self.process_decoded_udp_notif_packet(content_id.as_ref(), &subscription_info, &session, &pkt) { Ok(telemetry_message) => { - if let Err(err) = self.enriched_tx.send((content_id, subscription_info, telemetry_message)).await { + if let Err(err) = self.enriched_tx.send(EnrichedNotification { + cached_content_id: content_id, + subscription_info, + session, + message: telemetry_message, + }).await { error!("YangPushEnrichmentActor send error: {err}"); self.stats.send_error.add(1, &peer_tags); } else { @@ -519,18 +530,13 @@ impl std::error::Error for YangPushEnrichmentActorHandleError {} pub struct YangPushEnrichmentActorHandle { cmd_send: mpsc::Sender, enrichment_tx: async_channel::Sender, - enriched_rx: - async_channel::Receiver<(Option, SubscriptionInfo, TelemetryMessageWrapper)>, + enriched_rx: async_channel::Receiver, } impl YangPushEnrichmentActorHandle { pub fn new( buffer_size: usize, - validated_rx: async_channel::Receiver<( - Option, - SubscriptionInfo, - UdpNotifPacketDecoded, - )>, + validated_rx: async_channel::Receiver, manifest: Manifest, stats: either::Either, ) -> (JoinHandle>, Self) { @@ -565,10 +571,7 @@ impl YangPushEnrichmentActorHandle { .map_err(|_| YangPushEnrichmentActorHandleError::SendError) } - pub fn subscribe( - &self, - ) -> async_channel::Receiver<(Option, SubscriptionInfo, TelemetryMessageWrapper)> - { + pub fn subscribe(&self) -> async_channel::Receiver { self.enriched_rx.clone() } } @@ -625,7 +628,7 @@ mod tests { #[allow(clippy::type_complexity)] fn create_actor_handle() -> ( - async_channel::Sender<(Option, SubscriptionInfo, UdpNotifPacketDecoded)>, + async_channel::Sender, Manifest, JoinHandle>, YangPushEnrichmentActorHandle, @@ -655,7 +658,7 @@ mod tests { } fn create_subscription_started( - peer: SocketAddr, + peer_ip: IpAddr, id: SubscriptionId, ) -> (SubscriptionInfo, serde_json::Value, UdpNotifPacketDecoded) { let payload = json!({ @@ -679,7 +682,6 @@ mod tests { } }); - let collector = SocketAddr::from(([127, 0, 0, 1], 10000)); let packet = UdpNotifPacket::new( MediaType::YangDataJson, 1234, @@ -690,9 +692,7 @@ mod tests { let decoded: UdpNotifPacketDecoded = (&packet).try_into().unwrap(); let subscription_info = SubscriptionInfo::new( - collector, - None, - peer, + peer_ip, id, Target::new_datastore( DatastoreName::Operational.to_string(), @@ -722,21 +722,30 @@ mod tests { async fn test_process_payload_empty_subscription() { // Set up the enrichment actor and input test data let (msgs_tx, test_manifest, join_handle, actor_handle) = create_actor_handle(); - let collector = SocketAddr::from(([127, 0, 0, 1], 10000)); - let peer = SocketAddr::from(([127, 0, 0, 1], 12345)); - let (_subscription_info, json_payload, decoded) = create_subscription_started(peer, 1); - let empty_subscription_info = SubscriptionInfo::new_empty(collector, None, peer, 1); + let peer_ip = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); + let (_subscription_info, json_payload, decoded) = create_subscription_started(peer_ip, 1); + let empty_subscription_info = SubscriptionInfo::new_empty(peer_ip, 1); msgs_tx - .send(( - Some(empty_subscription_info.content_id().clone()), - empty_subscription_info.clone(), - decoded.clone(), - )) + .send(ValidatedNotification { + cached_content_id: Some(empty_subscription_info.content_id().clone()), + subscription_info: empty_subscription_info.clone(), + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(peer_ip, 0), + ), + packet: decoded.clone(), + }) .await .expect("Failed to send message to the actor"); tokio::task::yield_now().await; - let (received_content_id, received_subscription_info, received_enriched) = actor_handle + let EnrichedNotification { + cached_content_id: received_content_id, + subscription_info: received_subscription_info, + message: received_enriched, + .. + } = actor_handle .enriched_rx .recv() .await @@ -756,9 +765,9 @@ mod tests { None, SessionProtocol::YangPush, IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), - Some(12345), - None, - None, + Some(0), + Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))), + Some(10000), None, ), Some(test_manifest.clone()), @@ -785,19 +794,29 @@ mod tests { async fn test_process_payload_envelope() { // Set up the enrichment actor and input test data let (msgs_tx, test_manifest, join_handle, actor_handle) = create_actor_handle(); - let peer = SocketAddr::from(([127, 0, 0, 1], 12345)); - let (subscription_info, json_payload, decoded) = create_subscription_started(peer, 1); + let peer_ip = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); + let (subscription_info, json_payload, decoded) = create_subscription_started(peer_ip, 1); msgs_tx - .send(( - Some(subscription_info.content_id().clone()), - subscription_info.clone(), - decoded.clone(), - )) + .send(ValidatedNotification { + cached_content_id: Some(subscription_info.content_id().clone()), + subscription_info: subscription_info.clone(), + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(peer_ip, 0), + ), + packet: decoded.clone(), + }) .await .expect("Failed to send message to the actor"); tokio::task::yield_now().await; - let (received_content_id, received_subscription_info, received_enriched) = actor_handle + let EnrichedNotification { + cached_content_id: received_content_id, + subscription_info: received_subscription_info, + message: received_enriched, + .. + } = actor_handle .enriched_rx .recv() .await @@ -818,9 +837,9 @@ mod tests { None, SessionProtocol::YangPush, IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), - Some(12345), - None, - None, + Some(0), + Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))), + Some(10000), Some(expected_metadata), ), Some(test_manifest.clone()), @@ -845,8 +864,7 @@ mod tests { #[test] fn test_process_payload_envelope_without_content() { let mut actor = create_actor(); - let collector = SocketAddr::from(([127, 0, 0, 1], 10000)); - let peer = SocketAddr::from(([127, 0, 0, 1], 12345)); + let peer_ip = IpAddr::from([127, 0, 0, 1]); // Create a UdpNotifPayload without content let payload = json!({ @@ -867,11 +885,17 @@ mod tests { Bytes::from(payload), ); - let subscription_info = SubscriptionInfo::new_empty(collector, None, peer, 1); + let subscription_info = SubscriptionInfo::new_empty(peer_ip, 1); // Attempt to decode the packet (should succeed) let decoded: UdpNotifPacketDecoded = (&packet).try_into().unwrap(); + let session = SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(peer_ip, 0), + ); - let result = actor.process_decoded_udp_notif_packet(None, &subscription_info, &decoded); + let result = + actor.process_decoded_udp_notif_packet(None, &subscription_info, &session, &decoded); assert_eq!( result, diff --git a/crates/udp-notif-service/src/actor.rs b/crates/udp-notif-service/src/actor.rs index 0e784177..788ebcf5 100644 --- a/crates/udp-notif-service/src/actor.rs +++ b/crates/udp-notif-service/src/actor.rs @@ -115,8 +115,8 @@ //! ``` use crate::{ - ActorId, OTL_UDP_NOTIF_PUBLISHER_ID_KEY, SubscriberId, Subscription, UdpNotifPacket, - UdpNotifReceiver, UdpNotifRequest, UdpNotifSender, create_udp_notif_channel, + ActorId, OTL_UDP_NOTIF_PUBLISHER_ID_KEY, SessionInfo, SubscriberId, Subscription, + UdpNotifPacket, UdpNotifReceiver, UdpNotifRequest, UdpNotifSender, create_udp_notif_channel, }; use bytes::{Bytes, BytesMut}; use futures_util::StreamExt; @@ -645,9 +645,11 @@ impl UdpNotifActor { ); let mut send_handlers = vec![]; let request = Arc::new(UdpNotifRequest::new( - self.socket_addr, - self.interface_bind.clone().map(String::into_boxed_str), - peer, + SessionInfo::new( + self.socket_addr, + self.interface_bind.clone().map(String::into_boxed_str), + peer, + ), msg, )); for (id, tx) in &self.subscribers { @@ -1322,15 +1324,11 @@ mod tests { // Create references to compare with the received ones let ref1 = Ok(Ok(Arc::new(UdpNotifRequest::new( - handle.local_addr(), - None, - local_addr1, + SessionInfo::new(handle.local_addr(), None, local_addr1), pkt1.clone(), )))); let ref2 = Ok(Ok(Arc::new(UdpNotifRequest::new( - handle.local_addr(), - None, - local_addr2, + SessionInfo::new(handle.local_addr(), None, local_addr2), pkt2.clone(), )))); diff --git a/crates/udp-notif-service/src/lib.rs b/crates/udp-notif-service/src/lib.rs index 67425346..9794dcfa 100644 --- a/crates/udp-notif-service/src/lib.rs +++ b/crates/udp-notif-service/src/lib.rs @@ -38,38 +38,65 @@ pub type ActorId = u32; /// Type alias to that YANG-Push subscription ID as defined in [RFC8641](https://datatracker.ietf.org/doc/html/rfc8641) pub type SubscriberId = u32; -/// The UDP-Notif packet and the peer [SocketAddr] that sent it. +/// Transport session context for a UDP-Notif packet. +/// +/// - **collector**: The local socket address on which the collector received +/// the packet. +/// - **interface**: The network interface or VRF the collector socket is bound +/// to, if any. +/// - **peer**: The full remote socket address (IP + source port) of the sending +/// device. +#[derive(Debug, Clone, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +pub struct SessionInfo { + pub collector: SocketAddr, + pub interface: Option>, + pub peer: SocketAddr, +} + +impl SessionInfo { + pub fn new(collector: SocketAddr, interface: Option>, peer: SocketAddr) -> Self { + Self { + collector, + interface, + peer, + } + } + + pub const fn collector(&self) -> SocketAddr { + self.collector + } + + pub fn interface(&self) -> Option<&str> { + self.interface.as_deref() + } + + pub const fn peer(&self) -> SocketAddr { + self.peer + } +} + +/// The UDP-Notif packet together with the transport session it arrived on. #[derive(Debug, Clone, Eq, PartialEq)] pub struct UdpNotifRequest { - collector_address: SocketAddr, - collector_interface: Option>, - peer_address: SocketAddr, + session: SessionInfo, packet: UdpNotifPacket, } impl UdpNotifRequest { - pub fn new( - collector_address: SocketAddr, - collector_interface: Option>, - peer_address: SocketAddr, - packet: UdpNotifPacket, - ) -> Self { - Self { - collector_address, - collector_interface, - peer_address, - packet, - } + pub fn new(session: SessionInfo, packet: UdpNotifPacket) -> Self { + Self { session, packet } + } + pub const fn session(&self) -> &SessionInfo { + &self.session } - pub const fn collector_address(&self) -> SocketAddr { - self.collector_address + self.session.collector } pub fn collector_interface(&self) -> Option<&str> { - self.collector_interface.as_deref() + self.session.interface.as_deref() } pub const fn peer_address(&self) -> SocketAddr { - self.peer_address + self.session.peer } pub const fn packet(&self) -> &UdpNotifPacket { &self.packet diff --git a/crates/yang-push/Cargo.toml b/crates/yang-push/Cargo.toml index a6f20159..4bc9bccd 100644 --- a/crates/yang-push/Cargo.toml +++ b/crates/yang-push/Cargo.toml @@ -22,6 +22,7 @@ netcalyx-udp-notif-service = { workspace = true } netcalyx-netconf-proto = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +strum = { workspace = true } strum_macros = { workspace = true } chrono = { workspace = true, default-features = true, features = ["serde"] } tokio = { workspace = true, default-features = false } @@ -29,7 +30,7 @@ tracing = { workspace = true } opentelemetry = { workspace = true, features = ["metrics", "trace", "logs"] } either = { workspace = true } async-channel = { workspace = true } -yang4 = { workspace = true } +yang5 = { workspace = true } rustc-hash = { workspace = true } quick-xml = { workspace = true } futures-util = { workspace = true } diff --git a/crates/yang-push/src/cache/actor.rs b/crates/yang-push/src/cache/actor.rs index e411090d..224eddd9 100644 --- a/crates/yang-push/src/cache/actor.rs +++ b/crates/yang-push/src/cache/actor.rs @@ -224,8 +224,8 @@ use crate::{ use futures_util::StreamExt; use futures_util::stream::FuturesUnordered; use netcalyx_netconf_proto::yang_push::types::SubscriptionId; +use netcalyx_udp_notif_service::SessionInfo; use rustc_hash::FxHashMap; -use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -305,22 +305,26 @@ pub enum CacheActorCommand { #[derive(Debug)] pub enum CacheLookupCommand { - LookupBySubscriptionInfo(SubscriptionInfo, async_channel::Sender), + LookupBySubscriptionInfo( + SubscriptionInfo, + SessionInfo, + async_channel::Sender, + ), - LookupBySubscriptionInfoOneShot(SubscriptionInfo, oneshot::Sender), + LookupBySubscriptionInfoOneShot( + SubscriptionInfo, + SessionInfo, + oneshot::Sender, + ), LookupBySubscriptionId { - collector: SocketAddr, - interface: Option, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, tx: async_channel::Sender, }, LookupBySubscriptionIdOneShot { - collector: SocketAddr, - interface: Option, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, tx: oneshot::Sender, }, @@ -339,37 +343,44 @@ pub enum CacheLookupCommand { impl std::fmt::Display for CacheLookupCommand { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::LookupBySubscriptionInfo(subscription_info, _) => { - write!(f, "lookup by subscription info {subscription_info}") + Self::LookupBySubscriptionInfo(subscription_info, session, _) => { + write!( + f, + "lookup by subscription info {subscription_info} from peer {}", + session.peer() + ) } - Self::LookupBySubscriptionInfoOneShot(subscription_info, _) => { + Self::LookupBySubscriptionInfoOneShot(subscription_info, session, _) => { write!( f, - "lookup by subscription info {subscription_info} (one shot)" + "lookup by subscription info {subscription_info} from peer {} (one shot)", + session.peer() ) } Self::LookupBySubscriptionId { - collector, - interface, - peer, + session, subscription_id, tx: _tx, } => { write!( f, - "lookup by subscription id {subscription_id} from peer {peer}, collector {collector}, interface {interface:?}", + "lookup by subscription id {subscription_id} from peer {}, collector {}, interface {:?}", + session.peer(), + session.collector(), + session.interface(), ) } Self::LookupBySubscriptionIdOneShot { - collector, - interface, - peer, + session, subscription_id, tx: _tx, } => { write!( f, - "lookup by subscription id {subscription_id} from peer {peer}, collector {collector}, interface {interface:?} (one shot)", + "lookup by subscription id {subscription_id} from peer {}, collector {}, interface {:?} (one shot)", + session.peer(), + session.collector(), + session.interface(), ) } Self::LookupByContentId(content_id, _) => { @@ -455,7 +466,7 @@ impl CacheActor { match sender.send(response).await { Ok(_) => { debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -466,7 +477,7 @@ impl CacheActor { } Err(err) => { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -525,7 +536,7 @@ impl CacheActor { match sender.send(response) { Ok(_) => { debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -536,7 +547,7 @@ impl CacheActor { } Err(_err) => { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -588,11 +599,7 @@ impl CacheActor { &[ opentelemetry::KeyValue::new( "network.peer.address", - format!("{}", subscription_info.peer().ip()), - ), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(subscription_info.peer().port().into()), + format!("{}", subscription_info.peer_ip()), ), opentelemetry::KeyValue::new( OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, @@ -610,7 +617,7 @@ impl CacheActor { ], ); warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -632,7 +639,7 @@ impl CacheActor { .await .map_err(|error| { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -641,7 +648,7 @@ impl CacheActor { ); }); debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -655,11 +662,7 @@ impl CacheActor { let otl_tags = [ opentelemetry::KeyValue::new( "network.peer.address", - format!("{}", subscription_info.peer().ip()), - ), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(subscription_info.peer().port().into()), + format!("{}", subscription_info.peer_ip()), ), opentelemetry::KeyValue::new( OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, @@ -695,7 +698,7 @@ impl CacheActor { let yang_lib_ref = match result { Ok(yang_lib_ref) => { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -705,7 +708,7 @@ impl CacheActor { } Err(err) => { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -718,9 +721,7 @@ impl CacheActor { // First, remove all pending requests for this subscription info that are // requested with full subscription info let empty = SubscriptionInfo::new_empty( - subscription_info.collector(), - subscription_info.interface(), - subscription_info.peer(), + subscription_info.peer_ip(), subscription_info.id(), ); let mut pending_senders = self @@ -755,11 +756,7 @@ impl CacheActor { Vec::from([ opentelemetry::KeyValue::new( "network.peer.address", - format!("{}", subscription_info.peer().ip()), - ), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(subscription_info.peer().port().into()), + format!("{}", subscription_info.peer_ip()), ), opentelemetry::KeyValue::new( OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, @@ -779,11 +776,11 @@ impl CacheActor { async fn process_request(&mut self, request: CacheLookupCommand) { match request { - CacheLookupCommand::LookupBySubscriptionInfo(subscription_info, sender) => { + CacheLookupCommand::LookupBySubscriptionInfo(subscription_info, session, sender) => { let otl_tags = Self::otl_tags_from_subscription_inf(&subscription_info); self.stats.requests_received.add(1, otl_tags.as_ref()); debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -795,7 +792,7 @@ impl CacheActor { match yang_lib_ref { Some(yang_lib_ref) => { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -819,7 +816,7 @@ impl CacheActor { if should_fetch { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -828,14 +825,15 @@ impl CacheActor { self.stats.device_fetch_request.add(1, &otl_tags); let job_result = tokio::time::timeout( self.fetcher_timeout, - self.fetcher.fetch(subscription_info.clone()), + self.fetcher + .fetch(subscription_info.clone(), session.clone()), ) .await; let job = match job_result { Ok(worker_result) => worker_result, Err(err) => { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -855,7 +853,7 @@ impl CacheActor { .record(self.workers_queue.len() as u64, &[]); } else { debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -869,11 +867,15 @@ impl CacheActor { } } } - CacheLookupCommand::LookupBySubscriptionInfoOneShot(subscription_info, sender) => { + CacheLookupCommand::LookupBySubscriptionInfoOneShot( + subscription_info, + session, + sender, + ) => { let mut otl_tags = Self::otl_tags_from_subscription_inf(&subscription_info); self.stats.requests_received.add(1, otl_tags.as_ref()); debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -885,7 +887,7 @@ impl CacheActor { .get_by_subscription_info(&subscription_info) { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -898,7 +900,8 @@ impl CacheActor { self.stats.device_fetch_request.add(1, &otl_tags); let worker_result = tokio::time::timeout( self.fetcher_timeout, - self.fetcher.fetch_blocking(subscription_info.clone()), + self.fetcher + .fetch_blocking(subscription_info.clone(), session.clone()), ) .await; @@ -924,18 +927,13 @@ impl CacheActor { Self::send_yang_lib_ref_oneshot(&subscription_info, yang_lib_ref, sender); } CacheLookupCommand::LookupBySubscriptionId { - collector, - interface, - peer, + session, subscription_id, tx, } => { + let peer_ip = session.peer().ip(); let otel_tags = [ - opentelemetry::KeyValue::new("network.peer.address", format!("{}", peer.ip())), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(peer.port().into()), - ), + opentelemetry::KeyValue::new("network.peer.address", format!("{peer_ip}")), opentelemetry::KeyValue::new( OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, opentelemetry::Value::I64(subscription_id.into()), @@ -943,29 +941,24 @@ impl CacheActor { ]; self.stats.requests_received.add(1, &otel_tags); debug!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "processing cache lookup by subscription id request" ); let response = self .schema_cache - .get_by_subscription_id(peer.ip(), subscription_id); + .get_by_subscription_id(peer_ip, subscription_id); if let Some((subscription_info, yang_lib_ref)) = response { self.stats.cache_hits.add(1, &otel_tags); Self::send_yang_lib_ref(&subscription_info, yang_lib_ref, tx).await; } else { warn!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "cache miss: subscription id not found in cache" ); self.stats.cache_misses.add(1, &otel_tags); - let subscription_info = SubscriptionInfo::new_empty( - collector, - interface.clone(), - peer, - subscription_id, - ); + let subscription_info = SubscriptionInfo::new_empty(peer_ip, subscription_id); let entry = self .pending_requests .entry(subscription_info.clone()) @@ -975,26 +968,22 @@ impl CacheActor { if should_fetch { info!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "cache miss: starting fetch from device by subscription id" ); self.stats.device_fetch_request.add(1, &otel_tags); let job_result = tokio::time::timeout( self.fetcher_timeout, - self.fetcher.fetch_by_subscription_id( - collector, - interface.clone().map(String::into_boxed_str), - peer, - subscription_id, - ), + self.fetcher + .fetch_by_subscription_id(session.clone(), subscription_id), ) .await; let job = match job_result { Ok(worker_result) => worker_result, Err(err) => { warn!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, router_content_id=subscription_info.clone().content_id().clone(), target=%subscription_info.target(), @@ -1014,7 +1003,7 @@ impl CacheActor { .record(self.workers_queue.len() as u64, &[]); } else { debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -1028,18 +1017,13 @@ impl CacheActor { } } CacheLookupCommand::LookupBySubscriptionIdOneShot { - collector, - interface, - peer, + session, subscription_id, tx, } => { + let peer_ip = session.peer().ip(); let mut otel_tags = vec![ - opentelemetry::KeyValue::new("network.peer.address", format!("{}", peer.ip())), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(peer.port().into()), - ), + opentelemetry::KeyValue::new("network.peer.address", format!("{peer_ip}")), opentelemetry::KeyValue::new( OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, opentelemetry::Value::I64(subscription_id.into()), @@ -1047,18 +1031,18 @@ impl CacheActor { ]; self.stats.requests_received.add(1, &otel_tags); debug!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "processing cache lookup by subscription id request (one shot)"); let response = self .schema_cache - .get_by_subscription_id(peer.ip(), subscription_id); + .get_by_subscription_id(peer_ip, subscription_id); if let Some((subscription_info, yang_lib_ref)) = response { self.stats.cache_hits.add(1, &otel_tags); Self::send_yang_lib_ref_oneshot(&subscription_info, yang_lib_ref, tx); } else { warn!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "cache miss: subscription id not found in cache" ); @@ -1067,12 +1051,8 @@ impl CacheActor { let worker_result = tokio::time::timeout( self.fetcher_timeout, - self.fetcher.fetch_by_subscription_id_blocking( - collector, - interface.clone().map(String::into_boxed_str), - peer, - subscription_id, - ), + self.fetcher + .fetch_by_subscription_id_blocking(session.clone(), subscription_id), ) .await; let worker_result = match worker_result { @@ -1086,12 +1066,7 @@ impl CacheActor { format!("{err}"), )); self.stats.device_fetch_failed.add(1, &otel_tags); - let empty = SubscriptionInfo::new_empty( - collector, - interface, - peer, - subscription_id, - ); + let empty = SubscriptionInfo::new_empty(peer_ip, subscription_id); Err(Box::new((empty.clone(), err.into()))) } }; @@ -1295,11 +1270,17 @@ pub(crate) mod tests { use std::path::Path; use std::time::Duration; - pub(crate) fn test_subscription_info() -> SubscriptionInfo { - SubscriptionInfo::new( + pub(crate) fn test_session_info() -> SessionInfo { + SessionInfo::new( SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 100)), 10000), None, - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 200)), 830), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 200)), 50000), + ) + } + + pub(crate) fn test_subscription_info() -> SubscriptionInfo { + SubscriptionInfo::new( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 200)), 1, Target::new_datastore( "ds:operational".to_string(), @@ -1437,6 +1418,7 @@ pub(crate) mod tests { .request_tx() .send(CacheLookupCommand::LookupBySubscriptionInfo( subscription_info.clone(), + test_session_info(), tx, )) .await @@ -1489,6 +1471,7 @@ pub(crate) mod tests { .request_tx() .send(CacheLookupCommand::LookupBySubscriptionInfo( subscription_info.clone(), + test_session_info(), tx, )) .await @@ -1517,6 +1500,7 @@ pub(crate) mod tests { .request_tx() .send(CacheLookupCommand::LookupBySubscriptionInfo( subscription_info.clone(), + test_session_info(), tx, )) .await @@ -1563,7 +1547,11 @@ pub(crate) mod tests { tasks.push_back(tokio::spawn(async move { let (tx, rx) = async_channel::unbounded(); h.request_tx() - .send(CacheLookupCommand::LookupBySubscriptionInfo(sub, tx)) + .send(CacheLookupCommand::LookupBySubscriptionInfo( + sub, + test_session_info(), + tx, + )) .await .unwrap(); tokio::time::timeout(Duration::from_secs(1), rx.recv()) @@ -1612,9 +1600,7 @@ pub(crate) mod tests { handle .request_tx() .send(CacheLookupCommand::LookupBySubscriptionId { - collector: subscription_info.collector(), - interface: subscription_info.interface(), - peer: subscription_info.peer(), + session: test_session_info(), subscription_id: subscription_info.id(), tx, }) @@ -1629,12 +1615,7 @@ pub(crate) mod tests { .expect("failed to receive response"); assert_eq!( response.subscription_info(), - &SubscriptionInfo::new_empty( - subscription_info.collector(), - subscription_info.interface(), - subscription_info.peer(), - subscription_info.id(), - ) + &SubscriptionInfo::new_empty(subscription_info.peer_ip(), subscription_info.id(),) ); assert_eq!(response.yang_lib_ref(), None); @@ -1647,9 +1628,7 @@ pub(crate) mod tests { assert_eq!(hits_counts.len(), 1); assert_eq!( hits_counts.get(&SubscriptionInfo::new_empty( - subscription_info.collector(), - subscription_info.interface(), - subscription_info.peer(), + subscription_info.peer_ip(), subscription_info.id() )), Some(&1) @@ -1684,9 +1663,11 @@ pub(crate) mod tests { handle .request_tx() .send(CacheLookupCommand::LookupBySubscriptionId { - collector: subscription_info.collector(), - interface: subscription_info.interface(), - peer: subscription_info.peer(), + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(subscription_info.peer_ip(), 0), + ), subscription_id: subscription_info.id(), tx, }) @@ -1716,6 +1697,7 @@ pub(crate) mod tests { .request_tx() .send(CacheLookupCommand::LookupBySubscriptionInfo( subscription_info.clone(), + test_session_info(), tx, )) .await @@ -1763,9 +1745,11 @@ pub(crate) mod tests { let (tx, rx) = async_channel::unbounded(); h.request_tx() .send(CacheLookupCommand::LookupBySubscriptionId { - collector: sub.collector(), - interface: sub.interface(), - peer: sub.peer(), + session: SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(sub.peer_ip(), 0), + ), subscription_id: sub.id(), tx, }) diff --git a/crates/yang-push/src/cache/fetcher.rs b/crates/yang-push/src/cache/fetcher.rs index f6bca478..6c30570e 100644 --- a/crates/yang-push/src/cache/fetcher.rs +++ b/crates/yang-push/src/cache/fetcher.rs @@ -35,13 +35,14 @@ use netcalyx_netconf_proto::yang_push::subscription::{ }; use netcalyx_netconf_proto::yang_push::types::SubscriptionId; use netcalyx_netconf_proto::yanglib::{DatastoreName, PermissiveVersionChecker, YangLibrary}; +use netcalyx_udp_notif_service::SessionInfo; use rand::RngExt; use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tokio::task::JoinHandle; -use tracing::{error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; pub type FetcherResult = Result< (SubscriptionInfo, YangLibrary, HashMap, Box>), @@ -55,27 +56,25 @@ pub trait YangLibraryFetcher { fn fetch( &self, subscription_info: SubscriptionInfo, + session: SessionInfo, ) -> impl Future> + Send; /// A blocking version which returns directly the YANG library and schemas. fn fetch_blocking( &self, subscription_info: SubscriptionInfo, + session: SessionInfo, ) -> impl Future + Send; fn fetch_by_subscription_id( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> impl Future> + Send; fn fetch_by_subscription_id_blocking( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> impl Future + Send; } @@ -146,17 +145,18 @@ impl NetconfYangLibraryFetcher { async fn fetch_from_device( cfg: &FetchConfig, subscription_info: SubscriptionInfo, + session: SessionInfo, ) -> FetcherResult { - let collector = subscription_info.collector(); - let interface = subscription_info.interface(); - let peer = subscription_info.peer(); + let collector = session.collector(); + let interface = session.interface(); + let peer_ip = subscription_info.peer_ip(); let subscription_id = subscription_info.id(); - let host = SocketAddr::new(peer.ip(), cfg.default_port); + let host = SocketAddr::new(peer_ip, cfg.default_port); info!( host=%host, collector=%collector, interface, - peer=%peer, + peer_ip=%peer_ip, subscription_id, "starting fetching YANG Library from device", ); @@ -170,7 +170,7 @@ impl NetconfYangLibraryFetcher { auth, host, None, - interface, + interface.map(str::to_string), announce_caps, ssh_handler, Arc::clone(&cfg.client_config), @@ -208,7 +208,7 @@ impl NetconfYangLibraryFetcher { } info!( host=%host, - peer=%peer, + peer_ip=%peer_ip, subscription_id, cached_content_id=yang_lib.content_id(), schema_count=schemas.len(), @@ -219,17 +219,18 @@ impl NetconfYangLibraryFetcher { async fn fetch_from_device_by_id( cfg: &FetchConfig, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> FetcherResult { - let host = SocketAddr::new(peer.ip(), cfg.default_port); + let collector = session.collector(); + let interface = session.interface(); + let peer_ip = session.peer().ip(); + let host = SocketAddr::new(peer_ip, cfg.default_port); info!( host=%host, collector=%collector, interface, - peer=%peer, + peer_ip=%peer_ip, subscription_id, "starting fetching YANG Library from device", ); @@ -243,19 +244,14 @@ impl NetconfYangLibraryFetcher { auth, host, None, - interface.clone().map(String::from), + interface.map(String::from), announce_caps, ssh_handler, Arc::clone(&cfg.client_config), ); // Empty subscription info returned in case of errors to keep track of peer and // subscription ID - let empty = SubscriptionInfo::new_empty( - collector, - interface.clone().map(String::from), - peer, - subscription_id, - ); + let empty = SubscriptionInfo::new_empty(peer_ip, subscription_id); let mut client = match tokio::time::timeout(cfg.timeout, connect(config)).await { Ok(Ok(c)) => c, Ok(Err(err)) => { @@ -278,6 +274,12 @@ impl NetconfYangLibraryFetcher { .map_err(|err| Box::new((empty.clone(), err.into())))?; let modules = if let Some(modules) = &subscription.module_version { + debug!( + peer_ip=%peer_ip, + subscription_id, + modules=?modules, + "using module-version reported by device for subscription", + ); modules.clone().to_vec() } else { let (ds_name, namespaces) = match &subscription.target { @@ -312,15 +314,49 @@ impl NetconfYangLibraryFetcher { } }, }; + debug!( + peer_ip=%peer_ip, + subscription_id, + ds_name=?ds_name, + namespaces=?namespaces, + target=?subscription.target, + "no module-version reported by device, resolving target namespaces against YANG Library instead", + ); let mut ret = Vec::with_capacity(namespaces.len()); - for (_prefix, namespace) in namespaces { - let module = router_yang_library.find_module_by_datastore_and_ns(&ds_name, namespace).ok_or_else(|| Box::new((empty.clone(), YangLibraryCacheError::IoError(std::io::Error::other(format!("module with namespace {namespace} not found in YANG Library for datastore {ds_name}"))))))?; + for (prefix, namespace) in namespaces { + let module = router_yang_library.find_module_by_datastore_and_ns(&ds_name, namespace).ok_or_else(|| { + warn!( + peer_ip=%peer_ip, + subscription_id, + ds_name=?ds_name, + prefix, + namespace, + "module with namespace not found in YANG Library for datastore", + ); + Box::new((empty.clone(), YangLibraryCacheError::IoError(std::io::Error::other(format!("module with namespace {namespace} not found in YANG Library for datastore {ds_name}"))))) + })?; + trace!( + peer_ip=%peer_ip, + subscription_id, + prefix, + namespace, + module_name=module.name(), + "resolved xpath-filter prefix to module via namespace", + ); ret.push(YangPushModuleVersion::new( module.name().into(), module.revision().map(|x| x.into()), None, )); } + if ret.is_empty() { + warn!( + peer_ip=%peer_ip, + subscription_id, + target=?subscription.target, + "target namespaces resolution produced no modules; the target's YANG module(s) will not be fetched", + ); + } ret }; @@ -328,6 +364,12 @@ impl NetconfYangLibraryFetcher { if !module_names.contains(&"ietf-subscribed-notifications") { module_names.push("ietf-subscribed-notifications"); } + debug!( + peer_ip=%peer_ip, + subscription_id, + module_names=?module_names, + "final module list requested from device for subscription", + ); // TODO: add timeout to loading YANG Library from the device let (yang_lib, schemas) = client .load_from_modules(&module_names, &PermissiveVersionChecker) @@ -351,9 +393,7 @@ impl NetconfYangLibraryFetcher { )) })?; let subscription_info = SubscriptionInfo::new( - collector, - interface.map(String::from), - peer, + peer_ip, subscription_id, subscription_target, subscription.stop_time, @@ -366,7 +406,7 @@ impl NetconfYangLibraryFetcher { ); info!( host=%host, - peer=%peer, + peer_ip=%peer_ip, subscription_id, router_content_id=yang_lib.content_id(), target=%subscription_info.target(), @@ -382,7 +422,7 @@ impl NetconfYangLibraryFetcher { /// `operation` is called up to `retry.max_retries + 1` times. Each failed /// attempt waits `base * 2^(attempt-1)` (capped at `retry.max_backoff`) /// with equal jitter before the next try. - async fn with_retry(peer: SocketAddr, retry: RetryConfig, operation: F) -> FetcherResult + async fn with_retry(peer_ip: IpAddr, retry: RetryConfig, operation: F) -> FetcherResult where F: Fn() -> Fut, Fut: Future, @@ -396,7 +436,7 @@ impl NetconfYangLibraryFetcher { let jitter = rand::rng().random_range(0.0..=half); let delay = std::time::Duration::from_secs_f64(half + jitter); trace!( - %peer, + %peer_ip, attempt, delay_ms = delay.as_millis() as u64, "retrying YANG Library fetch after backoff", @@ -413,42 +453,43 @@ impl NetconfYangLibraryFetcher { } impl YangLibraryFetcher for NetconfYangLibraryFetcher { - async fn fetch(&self, subscription_info: SubscriptionInfo) -> JoinHandle { + async fn fetch( + &self, + subscription_info: SubscriptionInfo, + session: SessionInfo, + ) -> JoinHandle { let fetch_cfg = self.fetch_cfg.clone(); let retry_cfg = self.retry_cfg; tokio::spawn(async move { - Self::with_retry(subscription_info.peer(), retry_cfg, || { - Self::fetch_from_device(&fetch_cfg, subscription_info.clone()) + Self::with_retry(subscription_info.peer_ip(), retry_cfg, || { + Self::fetch_from_device(&fetch_cfg, subscription_info.clone(), session.clone()) }) .await }) } - async fn fetch_blocking(&self, subscription_info: SubscriptionInfo) -> FetcherResult { - Self::with_retry(subscription_info.peer(), self.retry_cfg, || { - Self::fetch_from_device(&self.fetch_cfg, subscription_info.clone()) + async fn fetch_blocking( + &self, + subscription_info: SubscriptionInfo, + session: SessionInfo, + ) -> FetcherResult { + Self::with_retry(subscription_info.peer_ip(), self.retry_cfg, || { + Self::fetch_from_device(&self.fetch_cfg, subscription_info.clone(), session.clone()) }) .await } async fn fetch_by_subscription_id( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> JoinHandle { let fetch_cfg = self.fetch_cfg.clone(); let retry_cfg = self.retry_cfg; + let peer_ip = session.peer().ip(); tokio::spawn(async move { - Self::with_retry(peer, retry_cfg, || { - Self::fetch_from_device_by_id( - &fetch_cfg, - collector, - interface.clone(), - peer, - subscription_id, - ) + Self::with_retry(peer_ip, retry_cfg, || { + Self::fetch_from_device_by_id(&fetch_cfg, session.clone(), subscription_id) }) .await }) @@ -456,19 +497,12 @@ impl YangLibraryFetcher for NetconfYangLibraryFetcher { async fn fetch_by_subscription_id_blocking( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> FetcherResult { - Self::with_retry(peer, self.retry_cfg, || { - Self::fetch_from_device_by_id( - &self.fetch_cfg, - collector, - interface.clone(), - peer, - subscription_id, - ) + let peer_ip = session.peer().ip(); + Self::with_retry(peer_ip, self.retry_cfg, || { + Self::fetch_from_device_by_id(&self.fetch_cfg, session.clone(), subscription_id) }) .await } @@ -493,7 +527,7 @@ pub(crate) mod tests { ) -> Self { for (subscription_info, (yang_lib, _schemas)) in &yang_libs { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -509,7 +543,7 @@ pub(crate) mod tests { fn get_from_cache(&self, subscription_info: SubscriptionInfo) -> FetcherResult { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -527,7 +561,7 @@ pub(crate) mod tests { .cloned() .ok_or_else(|| { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -542,23 +576,21 @@ pub(crate) mod tests { } fn get_from_cache_by_id(&self, subscription_info: SubscriptionInfo) -> FetcherResult { - let collector = subscription_info.collector(); - let interface = subscription_info.interface(); - let peer = subscription_info.peer(); + let peer_ip = subscription_info.peer_ip(); let subscription_id = subscription_info.id(); info!( - peer=%peer, + peer_ip=%peer_ip, subscription_id, "fetching from device by id" ); let subscription_info = self .yang_libs .keys() - .find(|x| x.id() == subscription_id && x.peer().ip() == peer.ip()); + .find(|x| x.id() == subscription_id && x.peer_ip() == peer_ip); let subscription_info = if let Some(subscription_info) = subscription_info { subscription_info.clone() } else { - SubscriptionInfo::new_empty(collector, interface, peer, subscription_id) + SubscriptionInfo::new_empty(peer_ip, subscription_id) }; // Increment counter in the instance state { @@ -577,7 +609,7 @@ pub(crate) mod tests { .cloned() .ok_or_else(|| { info!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -593,45 +625,41 @@ pub(crate) mod tests { } impl YangLibraryFetcher for TestYangLibFetcher { - async fn fetch(&self, subscription_info: SubscriptionInfo) -> JoinHandle { + async fn fetch( + &self, + subscription_info: SubscriptionInfo, + _session: SessionInfo, + ) -> JoinHandle { let result = self.get_from_cache(subscription_info); tokio::spawn(async move { result }) } - async fn fetch_blocking(&self, subscription_info: SubscriptionInfo) -> FetcherResult { + async fn fetch_blocking( + &self, + subscription_info: SubscriptionInfo, + _session: SessionInfo, + ) -> FetcherResult { self.get_from_cache(subscription_info) } async fn fetch_by_subscription_id( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> JoinHandle { - let subscription_info = SubscriptionInfo::new_empty( - collector, - interface.map(String::from), - peer, - subscription_id, - ); + let subscription_info = + SubscriptionInfo::new_empty(session.peer().ip(), subscription_id); let result = self.get_from_cache_by_id(subscription_info); tokio::spawn(async move { result }) } async fn fetch_by_subscription_id_blocking( &self, - collector: SocketAddr, - interface: Option>, - peer: SocketAddr, + session: SessionInfo, subscription_id: SubscriptionId, ) -> FetcherResult { - let subscription_info = SubscriptionInfo::new_empty( - collector, - interface.map(String::from), - peer, - subscription_id, - ); + let subscription_info = + SubscriptionInfo::new_empty(session.peer().ip(), subscription_id); self.get_from_cache_by_id(subscription_info) } } @@ -647,22 +675,18 @@ mod retry_tests { RetryConfig::new(max_retries, std::time::Duration::from_millis(1)) } - fn collector() -> SocketAddr { - SocketAddr::from(([127, 0, 0, 1], 10000)) - } - - fn dummy_peer() -> SocketAddr { - "127.0.0.1:0".parse().unwrap() + fn dummy_peer_ip() -> IpAddr { + "127.0.0.1".parse().unwrap() } fn make_ok() -> FetcherResult { - let info = SubscriptionInfo::new_empty(collector(), None, dummy_peer(), 1); + let info = SubscriptionInfo::new_empty(dummy_peer_ip(), 1); let yang_lib = YangLibrary::new("test-content-id".into(), vec![], vec![], vec![]); Ok((info, yang_lib, HashMap::new())) } fn make_err(msg: &'static str) -> FetcherResult { - let info = SubscriptionInfo::new_empty(collector(), None, dummy_peer(), 1); + let info = SubscriptionInfo::new_empty(dummy_peer_ip(), 1); Err(Box::new(( info, YangLibraryCacheError::IoError(std::io::Error::other(msg)), @@ -676,7 +700,7 @@ mod retry_tests { let call_count = Arc::new(AtomicU32::new(0)); let cc = Arc::clone(&call_count); - let result = NetconfYangLibraryFetcher::with_retry(dummy_peer(), retry_cfg(5), || { + let result = NetconfYangLibraryFetcher::with_retry(dummy_peer_ip(), retry_cfg(5), || { let cc = Arc::clone(&cc); async move { cc.fetch_add(1, Ordering::SeqCst); @@ -705,7 +729,7 @@ mod retry_tests { let call_count = Arc::new(AtomicU32::new(0)); let cc = Arc::clone(&call_count); - let result = NetconfYangLibraryFetcher::with_retry(dummy_peer(), retry_cfg(0), || { + let result = NetconfYangLibraryFetcher::with_retry(dummy_peer_ip(), retry_cfg(0), || { let cc = Arc::clone(&cc); async move { cc.fetch_add(1, Ordering::SeqCst); @@ -736,7 +760,7 @@ mod retry_tests { const MAX_RETRIES: u32 = 3; let result = - NetconfYangLibraryFetcher::with_retry(dummy_peer(), retry_cfg(MAX_RETRIES), || { + NetconfYangLibraryFetcher::with_retry(dummy_peer_ip(), retry_cfg(MAX_RETRIES), || { let cc = Arc::clone(&cc); async move { let n = cc.fetch_add(1, Ordering::SeqCst); @@ -769,7 +793,7 @@ mod retry_tests { let cc = Arc::clone(&call_count); const FAIL_FIRST: u32 = 2; // fail twice, succeed on the 3rd call - let result = NetconfYangLibraryFetcher::with_retry(dummy_peer(), retry_cfg(5), || { + let result = NetconfYangLibraryFetcher::with_retry(dummy_peer_ip(), retry_cfg(5), || { let cc = Arc::clone(&cc); async move { let n = cc.fetch_add(1, Ordering::SeqCst); diff --git a/crates/yang-push/src/cache/storage.rs b/crates/yang-push/src/cache/storage.rs index d758623e..c00ad316 100644 --- a/crates/yang-push/src/cache/storage.rs +++ b/crates/yang-push/src/cache/storage.rs @@ -106,7 +106,7 @@ use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::io::Write; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::Arc; use tracing::{debug, info, trace, warn}; @@ -727,15 +727,8 @@ impl YangLibraryReference { /// /// # Fields /// -/// - **collector**: The socket address (IP and port) of the YANG-Push collector -/// that received the subscription. -/// -/// - **interface**: The network interface name (if available) on which the -/// subscription was received. -/// -/// - **peer**: The socket address (IP and port) of the remote device that -/// established the subscription. This identifies which network element sent -/// the subscription-started notification. +/// - **peer_ip**: The IP address of the remote device that established the +/// subscription. /// /// - **content_id**: The YANG Library content identifier associated with this /// subscription. This links the subscription to a specific version of the @@ -761,31 +754,36 @@ impl YangLibraryReference { /// /// # Example /// -/// ```rust,ignore -/// use std::net::SocketAddr; +/// ```rust +/// use netcalyx_udp_notif_pkt::notification::Target; +/// use netcalyx_yang_push::cache::storage::SubscriptionInfo; +/// use std::net::IpAddr; /// /// let subscription_info = SubscriptionInfo::new( -/// "192.168.1.100:830".parse().unwrap(), -/// 1, -/// ContentId::from("2024-01-15-content-id"), +/// "192.168.1.1".parse::().unwrap(), // peer_ip +/// 1, // subscription id /// Target::new_datastore( /// "ds:operational".to_string(), /// either::Right("/ietf-interfaces:interfaces/ietf-interfaces/statistics".to_string()), /// ), -/// vec!["ietf-interfaces".to_string(), "ietf-ip".to_string()], +/// None, +/// None, +/// None, +/// None, +/// None, +/// Box::new([]), +/// "2024-01-15-content-id".to_string(), /// ); /// /// // Access subscription details -/// println!("Peer: {}", subscription_info.peer()); +/// println!("Peer IP: {}", subscription_info.peer_ip()); /// println!("Content ID: {}", subscription_info.content_id()); /// println!("Subscription Target: {}", subscription_info.target()); /// println!("Models: {:?}", subscription_info.models()); /// ``` #[derive(Clone, Debug, Eq, Hash, PartialEq, serde::Serialize, serde::Deserialize)] pub struct SubscriptionInfo { - pub collector: SocketAddr, - pub interface: Option, - pub peer: SocketAddr, + pub peer_ip: IpAddr, pub id: SubscriptionId, pub target: Target, @@ -806,15 +804,14 @@ pub struct SubscriptionInfo { pub models: Box<[YangPushModuleVersion]>, + /// router content-id for the advertised YANG Library pub content_id: ContentId, } impl SubscriptionInfo { #[allow(clippy::too_many_arguments)] pub fn new( - collector: SocketAddr, - interface: Option, - peer: SocketAddr, + peer_ip: IpAddr, id: SubscriptionId, target: Target, stop_time: Option>, @@ -826,9 +823,7 @@ impl SubscriptionInfo { content_id: ContentId, ) -> Self { Self { - collector, - interface, - peer, + peer_ip, id, target, stop_time, @@ -844,16 +839,9 @@ impl SubscriptionInfo { /// Create an empty subscription info placeholder. /// This can be used when no subscription info is available. /// Or to indicate that no yang library is associated with the subscription. - pub fn new_empty( - collector: SocketAddr, - interface: Option, - peer: SocketAddr, - id: SubscriptionId, - ) -> Self { + pub fn new_empty(peer_ip: IpAddr, id: SubscriptionId) -> Self { Self { - collector, - interface, - peer, + peer_ip, id, target: Target::new_datastore("EMPTY".to_string(), either::Right("EMPTY".to_string())), stop_time: None, @@ -870,20 +858,9 @@ impl SubscriptionInfo { self.content_id == "EMPTY" } - /// The peer address of the device who sent the subscription started - /// message. - pub const fn peer(&self) -> SocketAddr { - self.peer - } - - /// The collector address of the subscription. - pub const fn collector(&self) -> SocketAddr { - self.collector - } - - /// The interface name if available. - pub fn interface(&self) -> Option { - self.interface.clone() + /// The IP address of the device who sent the subscription started message. + pub const fn peer_ip(&self) -> IpAddr { + self.peer_ip } /// The subscription ID associated with the subscription. This is a unique @@ -933,8 +910,8 @@ impl std::fmt::Display for SubscriptionInfo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "SubscriptionInfo {{ peer: {}, content_id: {}, target: {}, models: {:?} }}", - self.peer, self.content_id, self.target, self.models + "SubscriptionInfo {{ peer_ip: {}, content_id: {}, target: {}, models: {:?} }}", + self.peer_ip, self.content_id, self.target, self.models ) } } @@ -1024,7 +1001,7 @@ impl YangLibraryCache { cache_by_content_id.insert(content_id, Arc::clone(&yang_lib_ref)); for subscription_info in subscriptions_info { cache_by_subscription_id - .entry(subscription_info.peer().ip()) + .entry(subscription_info.peer_ip()) .or_default() .insert(subscription_info.id(), subscription_info.clone()); cache_by_subscription_info.insert(subscription_info, Arc::clone(&yang_lib_ref)); @@ -1065,7 +1042,7 @@ impl YangLibraryCache { self.cache_by_subscription_info .insert(subscription_info.clone(), Arc::clone(existing_ref)); self.cache_by_subscription_id - .entry(subscription_info.peer().ip()) + .entry(subscription_info.peer_ip()) .or_default() .insert(subscription_info.id(), subscription_info); } @@ -1086,7 +1063,7 @@ impl YangLibraryCache { self.cache_by_content_id .insert(content_id, Arc::clone(&yang_lib_ref)); self.cache_by_subscription_id - .entry(subscription_info.peer().ip()) + .entry(subscription_info.peer_ip()) .or_default() .insert(subscription_info.id(), subscription_info); Ok(yang_lib_ref) @@ -1132,12 +1109,12 @@ impl YangLibraryCache { // Step 2: Remove from (peer_ip, sub_id) -> SubscriptionInfo mapping if let Some(sub_map) = self .cache_by_subscription_id - .get_mut(&subscription_info.peer().ip()) + .get_mut(&subscription_info.peer_ip()) { sub_map.remove(&subscription_info.id()); if sub_map.is_empty() { self.cache_by_subscription_id - .remove(&subscription_info.peer().ip()); + .remove(&subscription_info.peer_ip()); } } @@ -1185,7 +1162,7 @@ impl YangLibraryCache { .get(subscription_info) .map(Arc::clone); debug!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -1244,9 +1221,7 @@ mod tests { fn create_test_subscription_info(content_id: &str) -> SubscriptionInfo { SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 101], 830)), + IpAddr::from([192, 168, 1, 101]), 1, Target::new_datastore( "ds:operational".to_string(), @@ -1454,8 +1429,7 @@ mod tests { #[test] #[tracing_test::traced_test] fn test_subscription_info_new() { - let collector = SocketAddr::from(([192, 168, 1, 100], 10000)); - let peer = SocketAddr::from(([192, 168, 1, 101], 12345)); + let peer_ip = IpAddr::from([192, 168, 1, 101]); let content_id = ContentId::from("content-123".to_string()); let target = Target::new_datastore( "ds:operational".to_string(), @@ -1467,9 +1441,7 @@ mod tests { YangPushModuleVersion::new("model2".into(), None, None), ]); let info = SubscriptionInfo::new( - collector, - None, - peer, + peer_ip, 1, Target::new_datastore( "ds:operational".to_string(), @@ -1488,7 +1460,7 @@ mod tests { content_id.clone(), ); - assert_eq!(info.peer(), peer); + assert_eq!(info.peer_ip(), peer_ip); assert_eq!(info.content_id(), &content_id); assert_eq!(info.target(), &target); assert_eq!(info.models(), models.as_slice()); @@ -1500,7 +1472,7 @@ mod tests { let info = create_test_subscription_info("test-id"); let display = format!("{info}"); - assert!(display.contains("192.168.1.101:830")); + assert!(display.contains("192.168.1.101")); assert!(display.contains("test-id")); assert!(display.contains("/ietf-interfaces:interfaces/ietf-interfaces:interface[ietf-interfaces:name='eth0']/statistics")); assert!(display.contains("ietf-interfaces")); @@ -1620,7 +1592,7 @@ mod tests { let cache = YangLibraryCache::from_disk(temp_dir.path().to_path_buf()).unwrap(); let subscription_info = create_test_subscription_info(content_id); - let peer_ip = subscription_info.peer().ip(); + let peer_ip = subscription_info.peer_ip(); let subscription_id = subscription_info.id(); // Test successful lookup @@ -1726,9 +1698,7 @@ mod tests { let subscription_info1 = create_test_subscription_info("content_id"); let subscription_info2 = SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 102], 830)), + IpAddr::from([192, 168, 1, 102]), 2, Target::new_datastore( "ds:operational".to_string(), @@ -1751,9 +1721,7 @@ mod tests { ContentId::from("content_id2".to_string()), ); let subscription_info3 = SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 103], 830)), + IpAddr::from([192, 168, 1, 103]), 2, Target::new_datastore( "ds:operational".to_string(), @@ -1834,9 +1802,7 @@ mod tests { // Create different subscription info with the same content_id let subscription_info2 = SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 102], 830)), + IpAddr::from([192, 168, 1, 102]), 2, Target::new_datastore( "ds:operational".to_string(), @@ -1891,9 +1857,7 @@ mod tests { // Create first subscription info let subscription_info1 = SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 101], 830)), + IpAddr::from([192, 168, 1, 101]), 1, Target::new_datastore( "ds:operational".to_string(), @@ -1944,9 +1908,7 @@ mod tests { // Create second subscription info with different peer and target but same // content_id let subscription_info2 = SubscriptionInfo::new( - SocketAddr::from(([192, 168, 1, 100], 10000)), - None, - SocketAddr::from(([192, 168, 1, 102], 830)), + IpAddr::from([192, 168, 1, 102]), 2, Target::new_datastore( "ds:operational".to_string(), @@ -1995,13 +1957,13 @@ mod tests { // Verify both subscription IDs can be retrieved let (sub1_info, sub1_ref) = cache - .get_by_subscription_id(subscription_info1.peer().ip(), subscription_info1.id()) + .get_by_subscription_id(subscription_info1.peer_ip(), subscription_info1.id()) .unwrap(); assert_eq!(sub1_info, subscription_info1); assert!(sub1_ref.is_some()); let (sub2_info, sub2_ref) = cache - .get_by_subscription_id(subscription_info2.peer().ip(), subscription_info2.id()) + .get_by_subscription_id(subscription_info2.peer_ip(), subscription_info2.id()) .unwrap(); assert_eq!(sub2_info, subscription_info2); assert!(sub2_ref.is_some()); diff --git a/crates/yang-push/src/lib.rs b/crates/yang-push/src/lib.rs index 2de7ff9d..bd48597a 100644 --- a/crates/yang-push/src/lib.rs +++ b/crates/yang-push/src/lib.rs @@ -45,7 +45,3 @@ pub const OTL_YANG_PUSH_SUBSCRIPTION_ROUTER_CONTENT_ID_KEY: &str = "netcalyx.udp.notif.yang.push.subscription.router_content_id"; pub const OTL_YANG_PUSH_CACHED_CONTENT_ID_KEY: &str = "netcalyx.udp.notif.yang.push.subscription.cached_content_id"; -const OTL_CACHE_DROP_REASON_KEY: &str = "netcalyx.udp.notif.yang.push.cache.drop.reason"; -const OTL_CACHE_DROP_REASON_SUBSCRIPTION_CACHE_FULL: &str = "subscription cache is full"; -const OTL_CACHE_DROP_REASON_PEER_CACHE_FULL: &str = "peer cache is full"; -const OTL_YANG_PUSH_DECODE_ERROR_ID_KEY: &str = "netcalyx.udp.notif.yang.push.decode.error"; diff --git a/crates/yang-push/src/validation/mod.rs b/crates/yang-push/src/validation/mod.rs index 7be10629..ebf97a0e 100644 --- a/crates/yang-push/src/validation/mod.rs +++ b/crates/yang-push/src/validation/mod.rs @@ -29,11 +29,13 @@ //! # async fn example() -> Result<(), Box> { //! let (rx, tx, cache_cmd_tx) = /* channel setup */; //! let (join_handle, handle) = ValidationActorHandle::new( -//! 1000, // max packets buffered per peer -//! 100, // max packets buffered per subscription -//! rx, // incoming UDP-Notif packets -//! tx, // validated packets output -//! cache_cmd_tx, // cache lookup commands +//! 100, // cache response channel buffer size +//! 1000, // max packets buffered per peer +//! 100, // max packets buffered per subscription +//! rx, // incoming UDP-Notif packets +//! tx, // validated packets output +//! cache_cmd_tx, // cache lookup commands +//! either::Either::Left(meter), // metrics //! )?; //! //! // Actor runs in background... @@ -68,7 +70,7 @@ //! //! - **Subscription Level**: Per-subscription state including: //! - `SubscriptionInfo`: Metadata from `SubscriptionStarted` -//! - `yang4::Context`: Loaded YANG schemas for validation +//! - `yang5::Context`: Loaded YANG schemas for validation //! - Buffered packets waiting for schema retrieval //! - Enforces `max_buffered_packets_per_subscription` limit //! @@ -88,13 +90,13 @@ //! //! The actor validates packets when YANG schemas are available: //! -//! - **Schema available**: Validates using `yang4` library +//! - **Schema available**: Validates using `yang5` library //! - Valid packets → forwarded with full `SubscriptionInfo` //! - Invalid packets → dropped with error logged //! //! - **Schema unavailable**: Forwards unvalidated -//! - Marked with empty `SubscriptionInfo` (content_id = "EMPTY") -//! - Downstream can detect and handle unvalidated packets +//! - `content_id` is `None`; downstream can detect and handle unvalidated +//! packets //! //! - **Schema loading failed**: Disables validation for subscription //! - All future packets forwarded unvalidated @@ -113,12 +115,14 @@ //! - Cache channel closed: Actor terminates (dependency failure) //! - Shutdown command received: Graceful termination +// TODO: extract the validation logic into a separate module +// (for consistency with other actors in the codebase and to allow unit testing +// without the actor runtime) + use crate::cache::actor::{CacheLookupCommand, CacheResponse}; use crate::cache::storage::SubscriptionInfo; use crate::{ - ContentId, OTL_CACHE_DROP_REASON_KEY, OTL_CACHE_DROP_REASON_PEER_CACHE_FULL, - OTL_CACHE_DROP_REASON_SUBSCRIPTION_CACHE_FULL, OTL_YANG_PUSH_DECODE_ERROR_ID_KEY, - OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_ROUTER_CONTENT_ID_KEY, + ContentId, OTL_YANG_PUSH_SUBSCRIPTION_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_ROUTER_CONTENT_ID_KEY, OTL_YANG_PUSH_SUBSCRIPTION_TARGET_KEY, }; use netcalyx_netconf_proto::yang_push::subscription::YangPushModuleVersion; @@ -126,23 +130,130 @@ use netcalyx_netconf_proto::yang_push::types::SubscriptionId; use netcalyx_udp_notif_pkt::decoded::{UdpNotifPacketDecoded, UdpNotifPayload}; use netcalyx_udp_notif_pkt::notification::{NotificationVariant, SubscriptionStartedModified}; use netcalyx_udp_notif_pkt::raw::UdpNotifPacket; -use netcalyx_udp_notif_service::{OTL_UDP_NOTIF_PUBLISHER_ID_KEY, UdpNotifRequest}; +use netcalyx_udp_notif_service::{OTL_UDP_NOTIF_PUBLISHER_ID_KEY, SessionInfo, UdpNotifRequest}; use rustc_hash::FxHashMap; use std::collections::VecDeque; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use strum::VariantNames; use tokio::sync::mpsc; use tracing::{debug, info, trace, warn}; -use yang4::data::{DataFormat, DataOperation, DataParserFlags, DataValidationFlags}; +use yang5::data::{DataFormat, DataOperation, DataParserFlags, DataValidationFlags}; + +// Attribute key shared by the `dropped` and `skipped` counters. +const REASON_KEY: &str = "reason"; + +/// Attribute values for the `reason` key on the `dropped` counter. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + strum_macros::Display, + strum_macros::VariantNames, + strum_macros::IntoStaticStr, +)] +#[strum(serialize_all = "snake_case")] +enum DropReason { + DecodeError, + BufferFullSubscription, + BufferFullPeer, + ValidationFailed, + IncompleteSubscriptionStarted, + NoSubscriptionId, + SendError, +} + +/// Attribute values for the `reason` key on the `skipped` counter. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + strum_macros::Display, + strum_macros::VariantNames, + strum_macros::IntoStaticStr, +)] +#[strum(serialize_all = "snake_case")] +enum SkipReason { + NoLibrary, + ContextFailed, + NoSubscriptionInfo, +} + +// Attribute key for the `cache_lookups` counter. +const CACHE_LOOKUP_BY_KEY: &str = "by"; + +/// Attribute values for the `by` key on the `cache_lookups` counter. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + strum_macros::Display, + strum_macros::VariantNames, + strum_macros::IntoStaticStr, +)] +#[strum(serialize_all = "snake_case")] +enum CacheLookupBy { + SubscriptionInfo, + SubscriptionId, +} +/// Per-subscription state held by the validation actor. +/// +/// The combination of `schema_fetch_pending`, `yang_ctx`, and +/// `cached_content_id` encodes the current schema-loading state: +/// +/// | `schema_fetch_pending` | `yang_ctx` | `cached_content_id` | Meaning | +/// |------------------------|------------|---------------------|---------| +/// | `true` | `None` | `None` | Fetch in-flight — waiting for cache actor response | +/// | `false` | `Some(..)` | `Some(..)` | Schema loaded, validation active | +/// | `false` | `None` | `Some(..)` | YANG library on disk but context creation failed, validation disabled ¹ | +/// | `false` | `None` | `None` | No YANG library available, validation disabled ² | +/// +/// ¹ The cache actor returned a `YangLibraryReference` (files already +/// on disk from a prior cache hit or NETCONF device fetch), but +/// `Context::new_from_yang_library_file` failed — e.g. corrupt or missing +/// schema files. Packets are forwarded unvalidated. +/// +/// ² The cache actor returned no `YangLibraryReference` — the NETCONF device +/// fetch failed or timed out. Packets are forwarded unvalidated. +/// +/// `schema_fetch_pending` is set to `true` when a `LookupBySubscriptionInfo` +/// or `LookupBySubscriptionId` request is sent and cleared to `false` when +/// the cache actor responds, regardless of whether a schema was found. While +/// it is `true`, duplicate packets for the same subscription are buffered +/// rather than forwarded unvalidated. #[derive(Debug)] struct CachedSubscription { cached_content_id: Option, subscription_info: SubscriptionInfo, - yang_ctx: Option, + yang_ctx: Option, buffered_packets: Vec>, + schema_fetch_pending: bool, } +impl CachedSubscription { + fn new(subscription_info: SubscriptionInfo) -> Self { + Self { + cached_content_id: None, + subscription_info, + yang_ctx: None, + buffered_packets: Vec::new(), + schema_fetch_pending: false, + } + } +} + +// TODO: entries here are never evicted by TTL/idleness (only replaced when a +// subscription's SubscriptionStarted info changes via +// `check_subscription_new`). A peer churning through many subscription IDs can +// grow this map unboundedly; consider a purge mechanism similar to +// `UdpNotifSupervisorHandle::purge_unused_peers`. #[derive(Debug, Default)] struct CachedPeerSubscriptions { subscriptions: FxHashMap, @@ -151,119 +262,156 @@ struct CachedPeerSubscriptions { #[derive(Debug, Clone)] pub struct ValidationStats { - pub messages_received: opentelemetry::metrics::Counter, - pub messages_decoding_success: opentelemetry::metrics::Counter, - pub messages_decoding_fail: opentelemetry::metrics::Counter, - pub cache_request_by_subscription_info: opentelemetry::metrics::Counter, - pub cache_request_by_subscription_id: opentelemetry::metrics::Counter, - pub buffered_packets: opentelemetry::metrics::Gauge, - pub buffer_drop: opentelemetry::metrics::Counter, - pub buffer_drain: opentelemetry::metrics::Counter, - pub cache_yang_ctx_created: opentelemetry::metrics::Counter, - pub cache_yang_ctx_invalid: opentelemetry::metrics::Counter, - pub cache_yang_ctx_empty: opentelemetry::metrics::Counter, - pub validation_success: opentelemetry::metrics::Counter, - pub validation_invalid: opentelemetry::metrics::Counter, - pub validation_malformed: opentelemetry::metrics::Counter, - pub validation_skip: opentelemetry::metrics::Counter, - pub messages_sent: opentelemetry::metrics::Counter, - pub messages_dropped: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages received for validation + pub received: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages decoded successfully + pub decoded: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages dropped for any reason, tagged with + /// `reason` (`decode_error` | `buffer_full_subscription` | + /// `buffer_full_peer` | `validation_failed` | + /// `incomplete_subscription_started` | `no_subscription_id` | `send_error`) + pub dropped: opentelemetry::metrics::Counter, + /// Number of YANG schema cache lookups issued, tagged with `by` + /// (`subscription_info` | `subscription_id`) + pub cache_lookups: opentelemetry::metrics::Counter, + /// Current number of YANG-Push messages buffered waiting for schemas to + /// arrive + pub buffered: opentelemetry::metrics::Gauge, + /// Number of YANG-Push messages popped out of the buffer and queued for the + /// validation step + pub buffer_drained: opentelemetry::metrics::Counter, + /// Number of libyang validation contexts successfully created from a cache + /// response + pub yang_context_loaded: opentelemetry::metrics::Counter, + /// Number of libyang validation contexts that failed to be created (e.g., + /// missing schema files) + pub yang_context_failed: opentelemetry::metrics::Counter, + /// Number of cache responses with no YANG library (schema loading from the + /// router failed) + pub yang_context_empty: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages that passed YANG schema validation + pub validated: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages forwarded without validation, tagged with + /// `reason` (`no_library` | `context_failed` | `no_subscription_info`) + pub skipped: opentelemetry::metrics::Counter, + /// Number of YANG-Push messages successfully forwarded to the next actor + pub sent: opentelemetry::metrics::Counter, + /// Current number of packets in the `pending_packets` deque (drained from + /// subscription hold-buffers, awaiting reprocessing after schema arrival). + pub pending: opentelemetry::metrics::Gauge, + /// Current number of distinct peer IPs tracked in the peer cache + pub cached_peers: opentelemetry::metrics::Gauge, + /// Current number of subscriptions tracked for each peer in the peer cache, + /// tagged with `network.peer.address` + pub cached_subscriptions: opentelemetry::metrics::Gauge, } impl ValidationStats { pub fn new(meter: opentelemetry::metrics::Meter) -> Self { - let messages_received = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.received") + let received = meter + .u64_counter("netcalyx.yang_push.validation.received") .with_description( "Number of YANG-Push messages received for validation (before decoding)", ) .build(); - let messages_decoding_success = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.decoded.success") - .with_description("Number of YANG-Push messages decoded successfully (UDP-Notif payload read successfully)") + let decoded = meter + .u64_counter("netcalyx.yang_push.validation.decoded") + .with_description("Number of YANG-Push messages decoded successfully") .build(); - let messages_decoding_fail = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.decoded.fail") - .with_description("Number of YANG-Push messages dropped because of decoding errors (Couldn't read UDP-Notif payload)") - .build(); - let cache_request_by_subscription_info = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.cache.requests.by.subscription_info") - .with_description("Number of cache requests by subscription info (from subscription-start or subscription-modified messages) to retrieve the schemas for YANG-Push subscriptions") + let dropped = meter + .u64_counter("netcalyx.yang_push.validation.dropped") + .with_description(format!( + "Number of YANG-Push messages dropped for any reason, \ + tagged with reason ({})", + DropReason::VARIANTS.join(" | ") + )) .build(); - let cache_request_by_subscription_id = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.cache.requests.by.subscription_id") - .with_description("Number of cache requests by Subscription ID to retrieve the schemas for YANG-Push subscriptions") + let cache_lookups = meter + .u64_counter("netcalyx.yang_push.validation.cache.lookups") + .with_description(format!( + "Number of YANG schema cache lookups issued, tagged with by ({})", + CacheLookupBy::VARIANTS.join(" | ") + )) .build(); - let buffered_packets = meter - .u64_gauge("netcalyx.collector.yang_push.validation.buffer.packets") + let buffered = meter + .u64_gauge("netcalyx.yang_push.validation.buffered") .with_description("Number of YANG-Push messages currently buffered waiting for schemas") .build(); - let buffer_drop = meter - .u64_counter("netcalyx.collector.yang_push.validation.buffer.drop") - .with_description("Number of YANG-Push messages dropped because the buffer is full") - .build(); - let buffer_drain = meter - .u64_counter("netcalyx.collector.yang_push.validation.buffer.drain") - .with_description("Number of YANG-Push messages popped out of the buffer and sent to the validation step") + let buffer_drained = meter + .u64_counter("netcalyx.yang_push.validation.buffer.drained") + .with_description( + "Number of YANG-Push messages popped out of the buffer and sent to the validation step", + ) .build(); - let cache_yang_ctx_created = meter - .u64_counter("netcalyx.collector.yang_push.validation.cache.yang.ctx.created") - .with_description("Number of libyang validation context that are successfully created") + let yang_context_loaded = meter + .u64_counter("netcalyx.yang_push.validation.yang.context.loaded") + .with_description("Number of libyang validation contexts successfully created") .build(); - let cache_yang_ctx_invalid = meter - .u64_counter("netcalyx.collector.yang_push.validation.cache.yang.ctx.invalid") + let yang_context_failed = meter + .u64_counter("netcalyx.yang_push.validation.yang.context.failed") .with_description( - "Number of libyang validation context that are invalid (e.g., missing schema)", + "Number of libyang validation contexts that failed to be created (e.g., missing schema)", ) .build(); - let cache_yang_ctx_empty = meter - .u64_counter("netcalyx.collector.yang_push.validation.cache.yang.ctx.empty") - .with_description("Number of libyang validation context that are empty (e.g., schema loading from the router failed)") - .build(); - let validation_malformed = meter - .u64_counter("netcalyx.collector.yang_push.validation.malformed") + let yang_context_empty = meter + .u64_counter("netcalyx.yang_push.validation.yang.context.empty") .with_description( - "Number of YANG-Push messages dropped because they are malformed; e.g., missing subscription info", + "Number of cache responses with no YANG library (schema loading from the router failed)", ) .build(); - let validation_success = meter - .u64_counter("netcalyx.collector.yang_push.validation.success") - .with_description("Number of YANG-Push messages successfully validated") + let validated = meter + .u64_counter("netcalyx.yang_push.validation.validated") + .with_description("Number of YANG-Push messages that passed YANG schema validation") .build(); - let validation_invalid = meter - .u64_counter("netcalyx.collector.yang_push.validation.invalid") - .with_description("Number of YANG-Push messages dropped because of validation errors") + let skipped = meter + .u64_counter("netcalyx.yang_push.validation.skipped") + .with_description(format!( + "Number of YANG-Push messages forwarded without validation, \ + tagged with reason ({})", + SkipReason::VARIANTS.join(" | ") + )) + .build(); + let sent = meter + .u64_counter("netcalyx.yang_push.validation.sent") + .with_description( + "Number of YANG-Push messages successfully forwarded to the next actor", + ) .build(); - let validation_skip = meter - .u64_counter("netcalyx.collector.yang_push.validation.skipped") - .with_description("Number of YANG-Push skipped the validation step because the subscription is not found in the cache") + let pending = meter + .u64_gauge("netcalyx.yang_push.validation.pending") + .with_description( + "Current number of packets in the pending deque, drained from subscription \ + hold-buffers and awaiting reprocessing. Together with 'buffered' it accounts \ + for all packets held in memory by the validation actor.", + ) .build(); - let messages_sent = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.sent") - .with_description("Number of Telemetry Messages successfully sent upstream") + let cached_peers = meter + .u64_gauge("netcalyx.yang_push.validation.cached_peers") + .with_description("Current number of distinct peer IPs tracked in the peer cache") .build(); - let messages_dropped = meter - .u64_counter("netcalyx.collector.yang_push.validation.messages.dropped") - .with_description("Number of Telemetry Messages failed to send upstream") + let cached_subscriptions = meter + .u64_gauge("netcalyx.yang_push.validation.cached_subscriptions") + .with_description( + "Current number of subscriptions tracked per peer in the peer cache, \ + tagged with network.peer.address. Sum gives the global total.", + ) .build(); Self { - messages_received, - messages_decoding_success, - messages_decoding_fail, - cache_request_by_subscription_info, - cache_request_by_subscription_id, - buffered_packets, - buffer_drop, - buffer_drain, - cache_yang_ctx_created, - cache_yang_ctx_invalid, - cache_yang_ctx_empty, - validation_success, - validation_invalid, - validation_malformed, - validation_skip, - messages_sent, - messages_dropped, + received, + decoded, + dropped, + cache_lookups, + buffered, + buffer_drained, + yang_context_loaded, + yang_context_failed, + yang_context_empty, + validated, + skipped, + sent, + pending, + cached_peers, + cached_subscriptions, } } } @@ -285,13 +433,34 @@ enum ValidationActorCommand { Shutdown, } +/// The output of the validation stage: a decoded UDP-Notif packet together +/// with the subscription identity, transport session, and the locally-computed +/// schema fingerprint used for validation. +/// +/// - `cached_content_id`: the SHA-256 fingerprint of the YANG library that was +/// used to validate the packet, or `None` when the packet was forwarded +/// without validation (schema unavailable or fetch failed). +/// - `subscription_info`: subscription identity — what this data stream is +/// about. +/// - `session`: transport session context — collector Socket Address, +/// interface/VRF and peer Socket Address +/// - `packet`: the decoded UDP-Notif payload ready for enrichment and +/// publishing. +#[derive(Debug)] +pub struct ValidatedNotification { + pub cached_content_id: Option, + pub subscription_info: SubscriptionInfo, + pub session: SessionInfo, + pub packet: UdpNotifPacketDecoded, +} + struct ValidationActor { max_buffered_packets_per_peer: usize, max_buffered_packets_per_subscription: usize, peer_cache: FxHashMap, cmd_rx: mpsc::Receiver, rx: async_channel::Receiver>, - tx: async_channel::Sender<(Option, SubscriptionInfo, UdpNotifPacketDecoded)>, + tx: async_channel::Sender, cache_cmd_tx: async_channel::Sender, cache_tx: async_channel::Sender, cache_rx: async_channel::Receiver, @@ -346,8 +515,6 @@ impl ValidationActor { fn get_subscription_info( &mut self, peer: SocketAddr, - collector: SocketAddr, - interface: Option, decoded: &UdpNotifPacketDecoded, ) -> Option<(SubscriptionInfo, Option>)> { let message_id = decoded.message_id(); @@ -371,14 +538,9 @@ impl ValidationActor { subscription_started, ) = notif_contents { - let subscription_info = if let Some(subscription_info) = self.build_subscription_info( - peer, - collector, - interface, - message_id, - publisher_id, - subscription_started, - ) { + let subscription_info = if let Some(subscription_info) = + self.build_subscription_info(peer, message_id, publisher_id, subscription_started) + { subscription_info } else { warn!( @@ -420,6 +582,9 @@ impl ValidationActor { } } + /// Attempt to add `message` to the per-subscription hold buffer, enforcing + /// both the per-subscription and per-peer packet limits. + /// Returns `true` if the packet was buffered, `false` if it was dropped. fn buffer_packet( &mut self, subscription_info: SubscriptionInfo, @@ -438,7 +603,7 @@ impl ValidationActor { .get(&subscription_id) .map(|s| s.buffered_packets.len()) .unwrap_or(0); - if sub_buffered_packets > self.max_buffered_packets_per_subscription { + if sub_buffered_packets >= self.max_buffered_packets_per_subscription { // drop the new packet, since the buffer is full warn!( peer=%peer, @@ -450,13 +615,13 @@ impl ValidationActor { "Buffer full for subscription, dropping new packet" ); peer_tags.push(opentelemetry::KeyValue::new( - OTL_CACHE_DROP_REASON_KEY, - OTL_CACHE_DROP_REASON_SUBSCRIPTION_CACHE_FULL, + REASON_KEY, + <&str>::from(DropReason::BufferFullSubscription), )); - self.stats.buffer_drop.add(1, &peer_tags); + self.stats.dropped.add(1, &peer_tags); return false; } - if peer_cache.total_buffered > self.max_buffered_packets_per_peer { + if peer_cache.total_buffered >= self.max_buffered_packets_per_peer { warn!( peer=%peer, message_id, @@ -466,22 +631,16 @@ impl ValidationActor { router_content_id=subscription_info.content_id(), "Buffer full for peer, dropping new packet"); peer_tags.push(opentelemetry::KeyValue::new( - OTL_CACHE_DROP_REASON_KEY, - OTL_CACHE_DROP_REASON_PEER_CACHE_FULL, + REASON_KEY, + <&str>::from(DropReason::BufferFullPeer), )); - self.stats.buffer_drop.add(1, &peer_tags); + self.stats.dropped.add(1, &peer_tags); return false; } - let subscription_cache = - peer_cache - .subscriptions - .entry(subscription_id) - .or_insert(CachedSubscription { - cached_content_id: None, - subscription_info: subscription_info.clone(), - yang_ctx: None, - buffered_packets: Vec::new(), - }); + let subscription_cache = peer_cache + .subscriptions + .entry(subscription_id) + .or_insert_with(|| CachedSubscription::new(subscription_info.clone())); trace!( peer=%peer, message_id, @@ -494,11 +653,13 @@ impl ValidationActor { subscription_cache.buffered_packets.push(message); peer_cache.total_buffered += 1; self.stats - .buffered_packets + .buffered .record(peer_cache.total_buffered as u64, &peer_tags); true } + /// Build the base OpenTelemetry tag set (peer address, port, publisher id) + /// for a given UDPNotif packet. fn peer_tags_from_packet( peer: SocketAddr, packet: &UdpNotifPacket, @@ -517,7 +678,9 @@ impl ValidationActor { ]) } - fn extend_peer_targs_with_subscription_info( + /// Append subscription-specific OpenTelemetry tags (id, target, + /// router content-id) to an existing tag vector. + fn extend_peer_tags_with_subscription_info( subscription_info: &SubscriptionInfo, peer_tags: &mut Vec, ) { @@ -535,10 +698,17 @@ impl ValidationActor { )); } + /// Decode a raw `UdpNotifPacket` into a `UdpNotifPacketDecoded`. + /// Returns `Err(())` and drops the packet on unsupported media type or + /// parse failure. + /// + /// Set `count_decoded` to `false` when reprocessing a buffered message to + /// avoid double-counting the `decoded` metric. fn decode_message( &mut self, peer: SocketAddr, packet: &UdpNotifPacket, + count_decoded: bool, ) -> Result { let message_id = packet.message_id(); let publisher_id = packet.publisher_id(); @@ -567,7 +737,9 @@ impl ValidationActor { "Decoded UDP-Notif payload, starting the validation step" ); } - self.stats.messages_decoding_success.add(1, &peer_tags); + if count_decoded { + self.stats.decoded.add(1, &peer_tags); + } Ok(decoded) } Err(err) => { @@ -579,10 +751,10 @@ impl ValidationActor { "Failed to decode UDP-Notif payload, dropping packet" ); peer_tags.push(opentelemetry::KeyValue::new( - OTL_YANG_PUSH_DECODE_ERROR_ID_KEY, - format!("{err}"), + REASON_KEY, + <&str>::from(DropReason::DecodeError), )); - self.stats.messages_decoding_fail.add(1, &peer_tags); + self.stats.dropped.add(1, &peer_tags); Err(()) } } @@ -594,12 +766,14 @@ impl ValidationActor { async fn process_udp_notif_msg( &mut self, message: Arc, + is_reprocessed: bool, ) -> Result<(), ValidationActorError> { let peer = message.peer_address(); let packet = message.packet(); + let session = message.session().clone(); // Step 1: decode the raw UDP-Notif payload. - let decoded = match self.decode_message(peer, packet) { + let decoded = match self.decode_message(peer, packet, !is_reprocessed) { Ok(decoded) => decoded, // Decoding errors are logged in the [Self::decode_message], and packets are dropped // here @@ -625,19 +799,14 @@ impl ValidationActor { } else { return Ok(()); }; - Self::extend_peer_targs_with_subscription_info(&subscription_info, &mut peer_tags); + Self::extend_peer_tags_with_subscription_info(&subscription_info, &mut peer_tags); // Step 3: validate against YANG schemas if available, skip otherwise. let peer_cache = self.peer_cache.entry(peer.ip()).or_default(); let subscription_cache = peer_cache .subscriptions .entry(subscription_info.id()) - .or_insert(CachedSubscription { - cached_content_id: None, - subscription_info: subscription_info.clone(), - yang_ctx: None, - buffered_packets: Vec::new(), - }); + .or_insert_with(|| CachedSubscription::new(subscription_info.clone())); let cached_content_id = if let Some(cached_content_id) = subscription_cache.cached_content_id.clone() && let Some(yang_ctx) = subscription_cache.yang_ctx.as_ref() @@ -651,15 +820,23 @@ impl ValidationActor { ¬ification_type, yang_ctx, is_legacy, + &self.stats, + &peer_tags, ); - // logging of error is handled in the [Self::validate_message] if validation_result.is_err() { - self.stats.validation_invalid.add(1, &peer_tags); return Ok(()); } - self.stats.validation_success.add(1, &peer_tags); Some(cached_content_id) } else { + let skip_reason = if subscription_info.is_empty() { + SkipReason::NoSubscriptionInfo + } else if subscription_cache.cached_content_id.is_some() { + // Library reference exists but context creation failed + SkipReason::ContextFailed + } else { + // No library available — device fetch failed + SkipReason::NoLibrary + }; trace!( peer=%peer, message_id, @@ -668,19 +845,26 @@ impl ValidationActor { router_content_id=subscription_info.content_id(), target=%subscription_info.target(), notification_type, + skip_reason = <&str>::from(skip_reason), "No YANG schemas found, skipping validation step", ); - self.stats.validation_skip.add(1, &peer_tags); + let mut skip_tags = peer_tags.clone(); + skip_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(skip_reason), + )); + self.stats.skipped.add(1, &skip_tags); None }; // Step 4: forward to the enrichment actor. self.tx - .send(( - cached_content_id.clone(), - subscription_info.clone(), - decoded, - )) + .send(ValidatedNotification { + cached_content_id: cached_content_id.clone(), + subscription_info: subscription_info.clone(), + session, + packet: decoded, + }) .await .map_err(|_| { warn!( @@ -694,10 +878,15 @@ impl ValidationActor { notification_type, "Failed to send UDP-Notif message for the next actor to process" ); - self.stats.messages_dropped.add(1, &peer_tags); + let mut drop_tags = peer_tags.clone(); + drop_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(DropReason::SendError), + )); + self.stats.dropped.add(1, &drop_tags); ValidationActorError::SendError })?; - self.stats.messages_sent.add(1, &peer_tags); + self.stats.sent.add(1, &peer_tags); trace!( peer=%peer, message_id, @@ -712,34 +901,29 @@ impl ValidationActor { Ok(()) } + /// Validate the raw packet payload against the loaded YANG context. + /// Returns `Err` and drops the packet on validation failure. + #[allow(clippy::too_many_arguments)] fn validate_message( packet: &UdpNotifPacket, peer: SocketAddr, subscription_info: &SubscriptionInfo, cached_content_id: ContentId, notification_type: &String, - yang_ctx: &yang4::context::Context, + yang_ctx: &yang5::context::Context, is_legacy: bool, - ) -> Result<(), yang4::Error> { - let mut peer_tags = Self::peer_tags_from_packet(peer, packet); - Self::extend_peer_targs_with_subscription_info(subscription_info, &mut peer_tags); + stats: &ValidationStats, + peer_tags: &[opentelemetry::KeyValue], + ) -> Result<(), yang5::Error> { let message_id = packet.message_id(); let publisher_id = packet.publisher_id(); - let mut envelope_ext = None; - if let Some(ietf_yo_notif) = yang_ctx.get_module_implemented("ietf-yp-notification") - && let Some(ext) = ietf_yo_notif.extensions().next() - { - envelope_ext = Some(ext); - } - if let Some(envelope_ext) = envelope_ext - && !is_legacy - { - let validation_result = yang4::data::DataTree::parse_ext_string( - &envelope_ext, + if !is_legacy { + let validation_result = yang5::data::DataTree::parse_string( + yang_ctx, packet.payload(), DataFormat::JSON, - DataParserFlags::STRICT, + DataParserFlags::STRICT | DataParserFlags::ANYDATA_STRICT, DataValidationFlags::PRESENT, ); if let Err(err) = validation_result { @@ -757,10 +941,18 @@ impl ValidationActor { notification_type, error=%err, packet=packet_payload, - "Failed to validate UDP-Notif payload using draft-ietf-netconf-notif-envelope, dropping packet" + "Failed to validate UDP-Notif payload, dropping packet" ); + + let mut drop_tags = peer_tags.to_vec(); + drop_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(DropReason::ValidationFailed), + )); + stats.dropped.add(1, &drop_tags); return Err(err); } + stats.validated.add(1, peer_tags); trace!( peer=%peer, message_id, @@ -770,11 +962,11 @@ impl ValidationActor { target=%subscription_info.target(), notification_type, cached_content_id, - "Successfully validated YANG-Push message using draft-ietf-netconf-notif-envelope", + "Successfully validated YANG-Push message", ); Ok(()) } else { - let validation_result = yang4::data::DataTree::parse_op_string( + let validation_result = yang5::data::DataTree::parse_op_string( yang_ctx, packet.payload(), DataFormat::JSON, @@ -790,8 +982,15 @@ impl ValidationActor { cached_content_id, notification_type, error=%err, "Failed to validate legacy UDP-Notif payload, dropping packet"); + let mut drop_tags = peer_tags.to_vec(); + drop_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(DropReason::ValidationFailed), + )); + stats.dropped.add(1, &drop_tags); return Err(err); } + stats.validated.add(1, peer_tags); trace!( peer=%peer, message_id, @@ -815,8 +1014,6 @@ impl ValidationActor { peer: SocketAddr, decoded: &UdpNotifPacketDecoded, ) -> Result, ValidationActorError> { - let collector = message.collector_address(); - let interface = message.collector_interface(); let packet = message.packet(); let mut peer_tags = Self::peer_tags_from_packet(peer, packet); let message_id = decoded.message_id(); @@ -826,11 +1023,46 @@ impl ValidationActor { .map(|x| x.to_string()) .unwrap_or("UNKNOWN".to_string()); - match self.get_subscription_info(peer, collector, interface.map(String::from), decoded) { + match self.get_subscription_info(peer, decoded) { Some((subscription_info, cached_content_id)) => { - Self::extend_peer_targs_with_subscription_info(&subscription_info, &mut peer_tags); - if cached_content_id.is_some() { - return Ok(Some(subscription_info)); + Self::extend_peer_tags_with_subscription_info(&subscription_info, &mut peer_tags); + + match cached_content_id { + Some(Some(_)) => { + // Schema is loaded → validate and forward immediately. + return Ok(Some(subscription_info)); + } + Some(None) => { + // Cache entry exists but schema not yet available. We distinguish: + // - fetch in-flight (schema_fetch_pending = true): buffer the packet so it + // is validated once the response arrives, instead of slipping through + // unvalidated. + // - fetch already completed with no schema (schema_fetch_pending = false): + // forward unvalidated as usual; no point buffering. + let fetch_pending = self + .peer_cache + .get(&peer.ip()) + .and_then(|c| c.subscriptions.get(&subscription_info.id())) + .map(|s| s.schema_fetch_pending) + .unwrap_or(false); + if fetch_pending { + trace!( + peer=%peer, + message_id, + publisher_id, + subscription_id=subscription_info.id(), + router_content_id=subscription_info.content_id(), + subscription_target=%subscription_info.target(), + notification_type, + "Schema fetch in-flight, buffering packet until response arrives", + ); + self.buffer_packet(subscription_info.clone(), message); + return Ok(None); + } + // Fetch already completed → fast path. + return Ok(Some(subscription_info)); + } + None => {} // no entry → fall through to send lookup + buffer } debug!( peer=%peer, @@ -842,12 +1074,24 @@ impl ValidationActor { notification_type, "Received new subscription sending lookup by subscription info request to the cache" ); - self.stats - .cache_request_by_subscription_info - .add(1, &peer_tags); + peer_tags.push(opentelemetry::KeyValue::new( + CACHE_LOOKUP_BY_KEY, + <&str>::from(CacheLookupBy::SubscriptionInfo), + )); + self.stats.cache_lookups.add(1, &peer_tags); + + // Mark the fetch as in-flight + self.peer_cache + .entry(peer.ip()) + .or_default() + .subscriptions + .entry(subscription_info.id()) + .or_insert_with(|| CachedSubscription::new(subscription_info.clone())) + .schema_fetch_pending = true; self.cache_cmd_tx .send(CacheLookupCommand::LookupBySubscriptionInfo( subscription_info.clone(), + message.session().clone(), self.cache_tx.clone(), )) .await @@ -869,23 +1113,27 @@ impl ValidationActor { } None => { let notif_contents = decoded.payload().notification_contents(); - // A subscription-started/modified that reached here failed to - // build SubscriptionInfo (e.g. missing module version). It will - // fail identically every time, so buffering it and re-fetching - // would loop forever. Drop it permanently. if matches!( notif_contents, Some(NotificationVariant::SubscriptionStarted(_)) | Some(NotificationVariant::SubscriptionModified(_)) ) { + // A subscription-started/modified that reached here failed to + // build SubscriptionInfo (e.g. missing module version). It will + // fail identically every time, so buffering it and re-fetching + // would loop forever. Drop it permanently. warn!( peer=%peer, message_id, publisher_id, notification_type, - "Malformed subscription started/modified (no usable subscription info), dropping packet" + "Incomplete subscription started/modified (no usable subscription info), dropping packet" ); - self.stats.validation_malformed.add(1, &peer_tags); + peer_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(DropReason::IncompleteSubscriptionStarted), + )); + self.stats.dropped.add(1, &peer_tags); return Ok(None); } let subscription_id = notif_contents.map(|x| x.subscription_id()); @@ -898,25 +1146,40 @@ impl ValidationActor { notification_type, "Received UDP-Notif packet without subscription info, \ caching the packet and looking up subscription info in cache"); - self.stats - .cache_request_by_subscription_id - .add(1, &peer_tags); - let subscription_info = SubscriptionInfo::new_empty( - collector, - interface.map(String::from), - peer, - subscription_id, - ); + peer_tags.push(opentelemetry::KeyValue::new( + CACHE_LOOKUP_BY_KEY, + <&str>::from(CacheLookupBy::SubscriptionId), + )); + self.stats.cache_lookups.add(1, &peer_tags); + let subscription_info = SubscriptionInfo::new_empty(peer.ip(), subscription_id); + + // Mark the fetch as in-flight + self.peer_cache + .entry(peer.ip()) + .or_default() + .subscriptions + .entry(subscription_info.id()) + .or_insert_with(|| CachedSubscription::new(subscription_info.clone())) + .schema_fetch_pending = true; self.cache_cmd_tx .send(CacheLookupCommand::LookupBySubscriptionId { - collector, - interface: interface.map(String::from), - peer, + session: message.session().clone(), subscription_id, tx: self.cache_tx.clone(), }) .await - .map_err(|_| ValidationActorError::CacheLookupSendError)?; + .map_err(|error| { + warn!( + peer=%peer, + message_id, + publisher_id, + subscription_id, + notification_type, + error=%error, + "Error sending lookup by subscription id request to the cache" + ); + ValidationActorError::CacheLookupSendError + })?; self.buffer_packet(subscription_info.clone(), message); return Ok(None); } @@ -927,35 +1190,36 @@ impl ValidationActor { notification_type, "Received UDP-Notif packet without subscription info nor subscription ID, dropping packet" ); - self.stats.validation_invalid.add(1, &peer_tags); + peer_tags.push(opentelemetry::KeyValue::new( + REASON_KEY, + <&str>::from(DropReason::NoSubscriptionId), + )); + self.stats.dropped.add(1, &peer_tags); Ok(None) } } } + /// Handle a cache lookup response: load (or clear) the YANG context for the + /// subscription, clear `schema_fetch_pending`, and drain the hold buffer + /// into `pending_packets` for validation. fn process_cache_response( &mut self, response: CacheResponse, ) -> Result<(), ValidationActorError> { let (cached_content_id, subscription_info, yang_lib_ref) = response.into(); - let mut otl_tags = Vec::from([ - opentelemetry::KeyValue::new( - "network.peer.address", - format!("{}", subscription_info.peer().ip()), - ), - opentelemetry::KeyValue::new( - "network.peer.port", - opentelemetry::Value::I64(subscription_info.peer().port().into()), - ), - ]); - Self::extend_peer_targs_with_subscription_info(&subscription_info, &mut otl_tags); + let mut otl_tags = Vec::from([opentelemetry::KeyValue::new( + "network.peer.address", + format!("{}", subscription_info.peer_ip()), + )]); + Self::extend_peer_tags_with_subscription_info(&subscription_info, &mut otl_tags); let peer_cache = if let Some(peer_cache) = - self.peer_cache.get_mut(&subscription_info.peer().ip()) + self.peer_cache.get_mut(&subscription_info.peer_ip()) { peer_cache } else { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -971,7 +1235,7 @@ impl ValidationActor { subscription_cache } else { warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), target=%subscription_info.target(), @@ -984,21 +1248,21 @@ impl ValidationActor { subscription_cache.subscription_info = subscription_info.clone(); if let Some(yang_lib_ref) = yang_lib_ref { let search_dir = yang_lib_ref.search_dir(); - let yang_ctx_result = yang4::context::Context::new_from_yang_library_file( + let yang_ctx_result = yang5::context::Context::new_from_yang_library_file( &yang_lib_ref.yang_library_path(), DataFormat::XML, &search_dir.as_path(), - yang4::context::ContextFlags::empty(), + yang5::context::ContextFlags::empty(), ); let yang_ctx = match yang_ctx_result { Ok(yang_ctx) => { - self.stats.cache_yang_ctx_created.add(1, &otl_tags); + self.stats.yang_context_loaded.add(1, &otl_tags); Some(yang_ctx) } Err(err) => { - self.stats.cache_yang_ctx_invalid.add(1, &otl_tags); + self.stats.yang_context_failed.add(1, &otl_tags); warn!( - peer=%subscription_info.peer(), + peer_ip=%subscription_info.peer_ip(), subscription_id=subscription_info.id(), router_content_id=subscription_info.content_id(), cached_content_id=yang_lib_ref.content_id(), @@ -1012,10 +1276,11 @@ impl ValidationActor { subscription_cache.cached_content_id = cached_content_id.clone(); subscription_cache.yang_ctx = yang_ctx; } else { - self.stats.cache_yang_ctx_empty.add(1, &otl_tags); + self.stats.yang_context_empty.add(1, &otl_tags); subscription_cache.cached_content_id = None; subscription_cache.yang_ctx = None; } + subscription_cache.schema_fetch_pending = false; let buffered_packets = std::mem::take(&mut subscription_cache.buffered_packets); let drained = buffered_packets.len(); // Update the per-peer counter while we still hold the peer_cache borrow. @@ -1025,8 +1290,8 @@ impl ValidationActor { let peer = message.peer_address(); let packet = message.packet(); let mut peer_tags = Self::peer_tags_from_packet(peer, packet); - Self::extend_peer_targs_with_subscription_info(&subscription_info, &mut peer_tags); - self.stats.buffer_drain.add(1, &peer_tags); + Self::extend_peer_tags_with_subscription_info(&subscription_info, &mut peer_tags); + self.stats.buffer_drained.add(1, &peer_tags); trace!( peer=%peer, message_id=packet.message_id(), @@ -1039,17 +1304,34 @@ impl ValidationActor { ); self.pending_packets.push_back(message); } + self.stats.buffered.record(remaining as u64, &otl_tags); + self.stats + .pending + .record(self.pending_packets.len() as u64, &[]); self.stats - .buffered_packets - .record(remaining as u64, &otl_tags); + .cached_peers + .record(self.peer_cache.len() as u64, &[]); + let peer_ip = subscription_info.peer_ip(); + let peer_sub_count = self + .peer_cache + .get(&peer_ip) + .map(|p| p.subscriptions.len()) + .unwrap_or(0); + self.stats.cached_subscriptions.record( + peer_sub_count as u64, + &[opentelemetry::KeyValue::new( + "network.peer.address", + format!("{peer_ip}"), + )], + ); Ok(()) } + /// Construct a `SubscriptionInfo` from a `SubscriptionStarted/Modified` + /// notification. Returns `None` if module-version is absent. fn build_subscription_info( &self, peer: SocketAddr, - collector: SocketAddr, - interface: Option, message_id: u32, publisher_id: u32, sub_started: &SubscriptionStartedModified, @@ -1078,9 +1360,7 @@ impl ValidationActor { }; Some(SubscriptionInfo::new( - collector, - interface, - peer, + peer.ip(), sub_started.id(), sub_started.target().clone(), sub_started.stop_time().cloned(), @@ -1096,6 +1376,8 @@ impl ValidationActor { )) } + /// Actor event loop. Runs until a shutdown command is received or a + /// fatal channel error occurs. async fn run(mut self) -> Result { info!("Starting YANG-Push validation actor"); loop { @@ -1131,8 +1413,11 @@ impl ValidationActor { } } Some(message) = async { self.pending_packets.pop_front() }, if !self.pending_packets.is_empty() => { - if let Err(err) = self.process_udp_notif_msg(message).await { - let err_msg = "YANG-Push validation actor cached packet processing unrecoverable error, shutting down"; + self.stats + .pending + .record(self.pending_packets.len() as u64, &[]); + if let Err(err) = self.process_udp_notif_msg(message, true).await { + let err_msg = "YANG-Push validation actor buffered packet processing unrecoverable error, shutting down"; warn!(error=%err, err_msg); return Ok(err_msg.to_string()); } @@ -1140,14 +1425,14 @@ impl ValidationActor { msg = self.rx.recv() => { match msg { Ok(msg) => { - self.stats.messages_received.add( + self.stats.received.add( 1, &Self::peer_tags_from_packet( msg.peer_address(), msg.packet(), ), ); - if let Err(err) = self.process_udp_notif_msg(msg).await { + if let Err(err) = self.process_udp_notif_msg(msg, false).await { let err_msg = "YANG-Push validation actor UDP-Notif processing unrecoverable error, shutting down"; warn!(error=%err, err_msg); return Ok(err_msg.to_string()); @@ -1184,7 +1469,7 @@ impl ValidationActorHandle { max_buffered_packets_per_peer: usize, max_buffered_packets_per_subscription: usize, rx: async_channel::Receiver>, - tx: async_channel::Sender<(Option, SubscriptionInfo, UdpNotifPacketDecoded)>, + tx: async_channel::Sender, cache_cmd_tx: async_channel::Sender, stats: either::Either, ) -> Result< @@ -1235,25 +1520,24 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - #[tokio::test] - #[tracing_test::traced_test] - async fn test_validation_actor_schema_fetched() { - // Setup caching actor + /// Spawns a full validation actor stack with default buffer sizes + /// (peer=1000, subscription=100) and 100-slot channels. Returns + /// everything a test needs. Use for tests that don't require + /// non-standard buffer or channel sizes. + #[allow(clippy::type_complexity)] + fn setup_validation_actor() -> ( + tokio::task::JoinHandle>, + crate::cache::actor::CacheActorHandle, + SubscriptionInfo, + Arc>>, + async_channel::Sender>, + async_channel::Receiver, + ValidationActorHandle, + ) { let (caching_join_handle, caching_handle, subscription_info, fetcher_count) = setup_actor_with_empty_cache(); - { - let hits_counts = fetcher_count - .lock() - .expect("Failed to lock fetcher counts") - .clone(); - assert_eq!(hits_counts.len(), 0); - } - - // Setup channels let (udp_notif_tx, udp_notif_rx) = async_channel::bounded(100); let (validated_tx, validated_rx) = async_channel::bounded(100); - - // Spawn validation actor let (_join_handle, handle) = ValidationActorHandle::new( 100, 1000, @@ -1266,9 +1550,100 @@ mod tests { ))), ) .expect("Failed to spawn validation actor"); + ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) + } + + /// Sends a SubscriptionStarted to load YANG schemas, drains the forwarded + /// result, then returns so that the caller can send data packets that will + /// be validated against the loaded context. + async fn setup_and_load_schema( + udp_notif_tx: &async_channel::Sender>, + validated_rx: &async_channel::Receiver, + peer: SocketAddr, + ) { + let payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2025-09-23T14:12:16.024Z", + "hostname": "test-router-01", + "sequence-number": 0, + "contents": { + "ietf-subscribed-notifications:subscription-started": { + "id": 1, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "purpose": "test subscription", + "ietf-distributed-notif:message-publisher-id": [ + 16843789 + ], + "ietf-yang-push-revision:module-version": [ + { + "name": "ietf-interfaces", + "revision": "2018-02-20" + } + ], + "ietf-yang-push-revision:yang-library-content-id": "test-content-id-1", + "ietf-yang-push:periodic": { + "period": 6000 + } + } + } + } + }); + let bytes = serde_json::to_vec(&payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 1, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + // Draining the validated SubscriptionStarted also serves as the + // synchronisation point: by the time it is forwarded the YANG context + // is fully loaded and ready for subsequent push-update packets. + let ValidatedNotification { + cached_content_id: content_id, + .. + } = tokio::time::timeout(Duration::from_secs(2), validated_rx.recv()) + .await + .expect("timeout waiting for SubscriptionStarted to be validated") + .unwrap(); + assert!( + content_id.is_some(), + "SubscriptionStarted must pass YANG validation" + ); + } - // Create a test peer address - let peer = subscription_info.peer(); + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_schema_fetched() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + assert_eq!(fetcher_count.lock().unwrap().len(), 0); + + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); let payload = serde_json::json!( { "ietf-yp-notification:envelope": { @@ -1313,9 +1688,7 @@ mod tests { // Send SubscriptionStarted packet udp_notif_tx .send(Arc::new(UdpNotifRequest::new( - SocketAddr::from(([127, 0, 0, 1], 10000)), - None, - peer, + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), subscription_started_packet, ))) .await @@ -1325,11 +1698,14 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; // Verify packet is validated - let (content_id, sub_info, _validated) = - tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) - .await - .expect("timeout waiting for response") - .unwrap(); + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) + .await + .expect("timeout waiting for response") + .unwrap(); assert!(content_id.is_some()); assert!(!sub_info.is_empty()); @@ -1351,66 +1727,48 @@ mod tests { #[tokio::test] #[tracing_test::traced_test] async fn test_validation_actor_schema_not_found() { - // Setup caching actor - let (caching_join_handle, caching_handle, subscription_info, fetcher_count) = - setup_actor_with_empty_cache(); - { - let hits_counts = fetcher_count - .lock() - .expect("Failed to lock fetcher counts") - .clone(); - assert_eq!(hits_counts.len(), 0); - } - - // Setup channels - let (udp_notif_tx, udp_notif_rx) = async_channel::bounded(100); - let (validated_tx, validated_rx) = async_channel::bounded(100); - - // Spawn validation actor - let (_join_handle, handle) = ValidationActorHandle::new( - 100, - 1000, - 100, - udp_notif_rx, - validated_tx, - caching_handle.request_tx(), - either::Right(ValidationStats::new(opentelemetry::global::meter( - "test_meter", - ))), - ) - .expect("Failed to spawn validation actor"); - - // Create a test peer address - let peer = subscription_info.peer(); + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + assert_eq!(fetcher_count.lock().unwrap().len(), 0); + + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); let payload = serde_json::json!( { - "ietf-yp-notification:envelope": { - "event-time": "2025-09-23T14:12:16.024Z", - "hostname": "ipf-zbl1327-r-daisy-48", - "sequence-number": 0, - "contents": { - "ietf-subscribed-notifications:subscription-started": { - "id": 2, - "ietf-yang-push:datastore": "ietf-datastores:operational", - "ietf-yang-push:datastore-xpath-filter": "/ietf-hardware:hardware", - "transport": "ietf-udp-notif-transport:udp-notif", - "encoding": "encode-json", - "ietf-distributed-notif:message-publisher-id": [ - 16843789 - ], - "ietf-yang-push-revision:module-version": [ - { - "name": "ietf-hardware", - "revision": "" - } - ], - "ietf-yang-push-revision:yang-library-content-id": "test-content-id-1", - "ietf-yang-push:periodic": { - "period": 6000 - } - } + "ietf-yp-notification:envelope": { + "event-time": "2026-04-21T13:31:27.134Z", + "hostname": "ipf-zbl1312-r-ap-01", + "sequence-number": 0, + "contents": { + "ietf-subscribed-notifications:subscription-started": { + "id": 9, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces/interface", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "ietf-distributed-notif:message-publisher-id": [ + 16974839 + ], + "ietf-yang-push-revision:module-version": [ + { + "name": "ietf-interfaces", + "revision": "2018-02-20" + } + ], + "ietf-yang-push-revision:yang-library-content-id": "1903509911", + "ietf-yang-push:periodic": { + "period": 6000, + "anchor-time": "2025-01-01T00:00:30Z" } + } } + } } ); let bytes = serde_json::to_vec(&payload).unwrap(); @@ -1425,9 +1783,7 @@ mod tests { // Send SubscriptionStarted packet udp_notif_tx .send(Arc::new(UdpNotifRequest::new( - SocketAddr::from(([127, 0, 0, 1], 10000)), - None, - peer, + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), subscription_started_packet, ))) .await @@ -1437,11 +1793,14 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; // Verify packet is not validated - let (content_id, sub_info, _validated) = - tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) - .await - .expect("timeout waiting for response") - .unwrap(); + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) + .await + .expect("timeout waiting for response") + .unwrap(); assert!(content_id.is_none()); assert!(!sub_info.is_empty()); @@ -1488,24 +1847,31 @@ mod tests { ) .expect("Failed to spawn validation actor"); - let peer = subscription_info.peer(); + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); let payload = serde_json::json!({ "ietf-yp-notification:envelope": { - "event-time": "2025-09-23T14:12:16.024Z", + "event-time": "2026-04-21T13:33:31.007Z", + "hostname": "test-router-01", + "sequence-number": 1, "contents": { - "ietf-subscribed-notifications:subscription-started": { + "ietf-yang-push:push-update": { "id": 1, - "ietf-yang-push:datastore": "ietf-datastores:operational", - "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces", - "transport": "ietf-udp-notif-transport:udp-notif", - "encoding": "encode-json", - "purpose": "test subscription", - "ietf-distributed-notif:message-publisher-id": [16843789], - "ietf-yang-push-revision:module-version": [ - {"name": "ietf-interfaces", "revision": "2018-02-20"} - ], - "ietf-yang-push-revision:yang-library-content-id": "test-content-id-1", - "ietf-yang-push:periodic": {"period": 6000} + "datastore-contents": { + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "GigabitEthernet0/0/0", + "type": "iana-if-type:ethernetCsmacd", + "enabled": true, + "admin-status": "up", + "oper-status": "up", + "if-index": 1, + "speed": "1000000000" + } + ] + } + }, + "ietf-distributed-notif:message-publisher-id": 16974839 } } } @@ -1520,9 +1886,7 @@ mod tests { for i in 0..N { udp_notif_tx .send(Arc::new(UdpNotifRequest::new( - SocketAddr::from(([127, 0, 0, 1], 10000)), - None, - peer, + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), UdpNotifPacket::new( MediaType::YangDataJson, 10, @@ -1562,49 +1926,40 @@ mod tests { /// spin and nothing would shut down cleanly. #[tokio::test] async fn test_validation_actor_malformed_subscription_started_dropped() { - let (caching_join_handle, caching_handle, subscription_info, fetcher_count) = - setup_actor_with_empty_cache(); - - let (udp_notif_tx, udp_notif_rx) = async_channel::bounded(100); - let (validated_tx, validated_rx) = async_channel::bounded(100); - - let (_join_handle, handle) = ValidationActorHandle::new( - 100, - 1000, - 100, - udp_notif_rx, - validated_tx, - caching_handle.request_tx(), - either::Right(ValidationStats::new(opentelemetry::global::meter( - "test_meter", - ))), - ) - .expect("Failed to spawn validation actor"); + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); // SubscriptionStarted WITHOUT module-version → build_subscription_info // returns None → must be dropped permanently. - let peer = subscription_info.peer(); + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); let payload = serde_json::json!({ - "ietf-yp-notification:envelope": { - "event-time": "2025-09-23T14:12:16.024Z", - "contents": { - "ietf-subscribed-notifications:subscription-started": { - "id": 103, - "ietf-yang-push:datastore": "ietf-datastores:operational", - "ietf-yang-push:datastore-xpath-filter": "/ietf-hardware:hardware", - "transport": "ietf-udp-notif-transport:udp-notif", - "encoding": "encode-json", - "ietf-yang-push:periodic": {"period": 6000} - } + "ietf-yp-notification:envelope": { + "event-time": "2025-09-23T14:12:16.024Z", + "contents": { + "ietf-subscribed-notifications:subscription-started": { + "id": 103, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-hardware:hardware", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "ietf-yang-push:periodic": { + "period": 6000 } + } } + } }); let bytes = serde_json::to_vec(&payload).unwrap(); udp_notif_tx .send(Arc::new(UdpNotifRequest::new( - SocketAddr::from(([127, 0, 0, 1], 10000)), - None, - peer, + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), UdpNotifPacket::new( MediaType::YangDataJson, 10, @@ -1628,4 +1983,667 @@ mod tests { caching_handle.shutdown().await.unwrap(); caching_join_handle.await.unwrap().unwrap(); } + + /// A well-formed push-update must be forwarded after strict YANG validation + /// succeeds. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_valid_push_update_passes() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + _, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + setup_and_load_schema(&udp_notif_tx, &validated_rx, peer).await; + + // Send a push-update with ietf-interfaces data. + let push_update_payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2026-04-21T13:33:31.007Z", + "hostname": "ipf-zbl1312-r-ap-01", + "sequence-number": 2, + "contents": { + "ietf-yang-push:push-update": { + "id": 1, + "datastore-contents": { + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "Virtual-Template0", + "type": "iana-if-type:ppp", + "enabled": true, + "link-up-down-trap-enable": "enabled", + "admin-status": "up", + "oper-status": "up", + "if-index": 1, + "speed": "64000" + }, + { + "name": "GigabitEthernet0/0/0", + "type": "iana-if-type:ethernetCsmacd", + "enabled": true, + "link-up-down-trap-enable": "enabled", + "admin-status": "up", + "oper-status": "up", + "if-index": 4, + "phys-address": "8C:E5:EF:B7:18:4E", + "speed": "1000000000", + } + ] + } + }, + "ietf-yp-observation:timestamp": "2026-04-21T13:33:30.665Z", + "ietf-yp-observation:point-in-time": "current-accounting", + "ietf-distributed-notif:message-publisher-id": 16974839 + } + } + } + }); + let bytes = serde_json::to_vec(&push_update_payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 2, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) + .await + .expect("timeout: valid push-update was not forwarded") + .unwrap(); + assert!( + content_id.is_some(), + "valid push-update must pass strict YANG validation" + ); + assert!(!sub_info.is_empty()); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// A packet with an unsupported media type (XML) cannot be decoded and must + /// be dropped immediately with a `decode_error` warning. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_unsupported_media_type_dropped() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + _, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + + // YangDataXml is not handled by UdpNotifPacketDecoded::try_from → + // UnsupportedMediaType. + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(subscription_info.peer_ip(), 0), + ), + UdpNotifPacket::new( + MediaType::YangDataXml, + 10, + 1, + HashMap::new(), + Bytes::from_static(b""), + ), + ))) + .await + .unwrap(); + + let res = tokio::time::timeout(Duration::from_millis(300), validated_rx.recv()).await; + assert!( + res.is_err(), + "packet with unsupported media type must be dropped" + ); + assert!(logs_contain("Failed to decode UDP-Notif payload")); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// A packet whose payload is not valid JSON cannot be decoded and must be + /// dropped with a `decode_error` warning. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_malformed_json_payload_dropped() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + _, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + + // YangDataJson with bytes that are not valid JSON → serde_json parse error. + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new( + SocketAddr::from(([127, 0, 0, 1], 10000)), + None, + SocketAddr::new(subscription_info.peer_ip(), 0), + ), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 1, + HashMap::new(), + Bytes::from_static(b"this is not json {{{"), + ), + ))) + .await + .unwrap(); + + let res = tokio::time::timeout(Duration::from_millis(300), validated_rx.recv()).await; + assert!( + res.is_err(), + "packet with malformed JSON payload must be dropped" + ); + assert!(logs_contain("Failed to decode UDP-Notif payload")); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// A push-update containing a typo in a field name (i.e. an unknown YANG + /// node) must be dropped by strict YANG validation and never forwarded. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_invalid_push_update_dropped() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + _, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + setup_and_load_schema(&udp_notif_tx, &validated_rx, peer).await; + + // Push-update with "enabelled" (typo for "enabled"): an unknown YANG node + // that strict validation must reject. + let invalid_push_update_payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2026-04-21T13:33:31.007Z", + "hostname": "test-router-01", + "sequence-number": 1, + "contents": { + "ietf-yang-push:push-update": { + "id": 1, + "datastore-contents": { + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "GigabitEthernet0/0/0", + "type": "iana-if-type:ethernetCsmacd", + "enabelled": true, + "admin-status": "up", + "oper-status": "up", + "if-index": 1, + "speed": "1000000000" + } + ] + } + }, + "ietf-distributed-notif:message-publisher-id": 16974839 + } + } + } + }); + let bytes = serde_json::to_vec(&invalid_push_update_payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 2, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + + let res = tokio::time::timeout(Duration::from_millis(300), validated_rx.recv()).await; + assert!( + res.is_err(), + "push-update with a typo in a field name must be dropped by strict YANG validation" + ); + assert!(logs_contain("Failed to validate UDP-Notif payload")); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + // TODO(libyang): mandatory-node enforcement inside `anydata` is not yet + // implemented upstream; once fixed, flip assertions to + // `res.is_err()` + `logs_contain`. + /// A push-update whose interface entry omits the mandatory `type` leaf must + /// be dropped by strict YANG validation and never forwarded. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_missing_mandatory_node_dropped() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + _, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + setup_and_load_schema(&udp_notif_tx, &validated_rx, peer).await; + + // Push-update with the mandatory `type` leaf absent from the interface + // entry: ietf-interfaces@2018-02-20 declares it `mandatory true`. + let missing_type_payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2026-04-21T13:33:31.007Z", + "hostname": "test-router-01", + "sequence-number": 1, + "contents": { + "ietf-yang-push:push-update": { + "id": 1, + "datastore-contents": { + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "GigabitEthernet0/0/0", + "enabled": true, + "admin-status": "up", + "oper-status": "up", + "if-index": 1, + "speed": "1000000000" + } + ] + } + }, + "ietf-distributed-notif:message-publisher-id": 16974839 + } + } + } + }); + let bytes = serde_json::to_vec(&missing_type_payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 2, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + + // TODO(libyang): should be `res.is_err()` once libyang enforces mandatory nodes + // inside anydata. + let res = tokio::time::timeout(Duration::from_millis(300), validated_rx.recv()).await; + assert!( + res.is_ok(), + "libyang limitation apparently addressed: mandatory nodes inside anydata are now \ + enforced; flip this test to assert `res.is_err()` + `logs_contain(\"Failed to validate\")`" + ); + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = res.unwrap().unwrap(); + assert!(content_id.is_some()); + assert!(!sub_info.is_empty()); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// A SubscriptionStarted duplicate or SubscriptionModified with equal + /// subscription information that arrives while the first schema fetch is + /// still in-flight must be buffered (not forwarded unvalidated). Both + /// packets must emerge from validated_rx with a valid content_id once + /// the schema arrives, and only one cache fetch may be triggered. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_duplicate_subscription_started_in_flight() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + + let payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2025-09-23T14:12:16.024Z", + "hostname": "test-router-01", + "sequence-number": 0, + "contents": { + "ietf-subscribed-notifications:subscription-modified": { + "id": 1, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "purpose": "test subscription", + "ietf-distributed-notif:message-publisher-id": [ + 16843789 + ], + "ietf-yang-push-revision:module-version": [ + { + "name": "ietf-interfaces", + "revision": "2018-02-20" + } + ], + "ietf-yang-push-revision:yang-library-content-id": "test-content-id-1", + "ietf-yang-push:periodic": { + "period": 6000 + } + } + } + } + }); + let bytes = serde_json::to_vec(&payload).unwrap(); + + let make_packet = |msg_id: u32| { + Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + msg_id, + HashMap::new(), + Bytes::from(bytes.clone()), + ), + )) + }; + + // Send first SubscriptionStarted. This triggers the schema fetch and + // sets schema_fetch_pending = true. + udp_notif_tx.send(make_packet(1)).await.unwrap(); + // Yield so the actor processes the first packet before the duplicate is queued. + tokio::task::yield_now().await; + + // Send the duplicate while the fetch is in-flight. With schema_fetch_pending = + // true the actor must buffer it rather than forwarding it unvalidated. + udp_notif_tx.send(make_packet(2)).await.unwrap(); + + // Both packets must eventually be forwarded with a valid content_id. + // Regardless of whether the duplicate arrived before or after the cache + // responded, content_id must be Some (never forwarded unvalidated). + for i in 1..=2u32 { + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(3), validated_rx.recv()) + .await + .unwrap_or_else(|_| panic!("timeout waiting for packet {i}")) + .unwrap(); + assert!( + content_id.is_some(), + "packet {i}: duplicate SubscriptionStarted must be validated, not forwarded unvalidated" + ); + assert!(!sub_info.is_empty()); + } + + // Exactly one cache fetch must have been triggered for both identical packets. + assert_eq!( + fetcher_count.lock().unwrap().len(), + 1, + "only one cache fetch must be triggered for duplicate identical SubscriptionStarted" + ); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// A SubscriptionStarted duplicate that arrives after the schema is already + /// loaded must be validated immediately using the cached context, without + /// triggering a second cache fetch. + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_duplicate_subscription_started_after_schema_loaded() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + + // Load the schema via the first SubscriptionStarted and drain the result. + setup_and_load_schema(&udp_notif_tx, &validated_rx, peer).await; + assert_eq!( + fetcher_count.lock().unwrap().len(), + 1, + "first fetch must have been triggered" + ); + + // Send an identical SubscriptionStarted now that the schema is cached. + let payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2025-09-23T14:12:16.024Z", + "hostname": "test-router-01", + "sequence-number": 0, + "contents": { + "ietf-subscribed-notifications:subscription-started": { + "id": 1, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "purpose": "test subscription", + "ietf-distributed-notif:message-publisher-id": [ + 16843789 + ], + "ietf-yang-push-revision:module-version": [ + { + "name": "ietf-interfaces", + "revision": "2018-02-20" + } + ], + "ietf-yang-push-revision:yang-library-content-id": "test-content-id-1", + "ietf-yang-push:periodic": { + "period": 6000 + } + } + } + } + }); + let bytes = serde_json::to_vec(&payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 2, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + + // Must be validated immediately using the cached schema. + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(1), validated_rx.recv()) + .await + .expect("timeout: duplicate SubscriptionStarted was not forwarded") + .unwrap(); + assert!( + content_id.is_some(), + "duplicate SubscriptionStarted after schema loaded must be validated" + ); + assert!(!sub_info.is_empty()); + + // No additional fetch must have been triggered. + assert_eq!( + fetcher_count.lock().unwrap().len(), + 1, + "no additional cache fetch must be triggered when schema is already cached" + ); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } + + /// When a SubscriptionStarted with changed params (same id, updated + /// yang-library-content-id) arrives after the schema is already loaded: + /// 1. The validation actor clears its local cache entry, buffers the + /// packet, and sends a new `LookupBySubscriptionInfo` to the cache + /// actor. + /// 2. The cache actor detects the content-id changed and issues a fresh + /// fetch from the device by calling the fetcher. + /// 3. The test fetcher only knows about "test-content-id-1"; for any other + /// content-id it returns an error (simulating a device that does not yet + /// have the schema, or a NETCONF fetch failure). The cache actor + /// therefore sends back `yang_lib_ref = None`. + /// 4. The validation actor drains the buffer: since no schema is available, + /// the packet is forwarded unvalidated (`content_id = None`). + /// 5. Two distinct device fetch calls must have been made (one per + /// content-id). + #[tokio::test] + #[tracing_test::traced_test] + async fn test_validation_actor_changed_subscription_started_triggers_refetch() { + let ( + caching_join_handle, + caching_handle, + subscription_info, + fetcher_count, + udp_notif_tx, + validated_rx, + handle, + ) = setup_validation_actor(); + let peer = SocketAddr::new(subscription_info.peer_ip(), 0); + + // Load the schema for the initial subscription (content-id = + // "test-content-id-1"). + setup_and_load_schema(&udp_notif_tx, &validated_rx, peer).await; + assert_eq!( + fetcher_count.lock().unwrap().len(), + 1, + "initial fetch must have fired once" + ); + + // Send a SubscriptionStarted for the same subscription id but with a + // different yang-library-content-id, simulating a schema change after a + // device software upgrade. + let changed_payload = serde_json::json!({ + "ietf-yp-notification:envelope": { + "event-time": "2025-09-24T08:00:00.000Z", + "hostname": "test-router-01", + "sequence-number": 1, + "contents": { + "ietf-subscribed-notifications:subscription-started": { + "id": 1, + "ietf-yang-push:datastore": "ietf-datastores:operational", + "ietf-yang-push:datastore-xpath-filter": "/ietf-interfaces:interfaces", + "transport": "ietf-udp-notif-transport:udp-notif", + "encoding": "encode-json", + "purpose": "test subscription", + "ietf-distributed-notif:message-publisher-id": [16843789], + "ietf-yang-push-revision:module-version": [ + {"name": "ietf-interfaces", "revision": "2018-02-20"} + ], + "ietf-yang-push-revision:yang-library-content-id": "updated-content-id-2", + "ietf-yang-push:periodic": {"period": 6000} + } + } + } + }); + let bytes = serde_json::to_vec(&changed_payload).unwrap(); + udp_notif_tx + .send(Arc::new(UdpNotifRequest::new( + SessionInfo::new(SocketAddr::from(([127, 0, 0, 1], 10000)), None, peer), + UdpNotifPacket::new( + MediaType::YangDataJson, + 10, + 2, + HashMap::new(), + Bytes::from(bytes), + ), + ))) + .await + .unwrap(); + + // The test fetcher only knows "test-content-id-1" and returns an error for + // "updated-content-id-2" (simulating a failed device fetch). The packet was + // buffered during the fetch attempt; after the fetch fails it is forwarded + // unvalidated. In production the fetch would succeed and content_id would be + // Some. + let ValidatedNotification { + cached_content_id: content_id, + subscription_info: sub_info, + .. + } = tokio::time::timeout(Duration::from_secs(3), validated_rx.recv()) + .await + .expect("timeout: changed SubscriptionStarted was not forwarded") + .unwrap(); + assert!( + content_id.is_none(), + "device fetch failed for new content-id → packet must be forwarded unvalidated" + ); + assert!(!sub_info.is_empty()); + + // A second device fetch must have been triggered for the new content-id; + // the cache must not silently reuse the old schema when content-id changes. + assert_eq!( + fetcher_count.lock().unwrap().len(), + 2, + "a new device fetch must be triggered when yang-library-content-id changes" + ); + + handle.shutdown().await.unwrap(); + caching_handle.shutdown().await.unwrap(); + caching_join_handle.await.unwrap().unwrap(); + } }