From 595e033776db24ee308b1841f4c00e49f7790a3c Mon Sep 17 00:00:00 2001
From: Grant Limberg <glimberg@users.noreply.github.com>
Date: Fri, 28 Apr 2023 14:24:19 -0700
Subject: [PATCH] Outgoing Packet Metrics (#1980)
add tx/rx labels to packet counters and add metrics for outgoing packets
---
node/IncomingPacket.cpp | 55 ++++++------
node/Metrics.cpp | 179 ++++++++++++++++++++++++++++------------
node/Metrics.hpp | 93 +++++++++++++++------
node/Node.cpp | 5 +-
node/Peer.cpp | 1 +
node/Switch.cpp | 73 +++++++++++++++-
node/Switch.hpp | 1 +
7 files changed, 297 insertions(+), 110 deletions(-)
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 12534117f..532a99fe0 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -86,6 +86,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr,int32_t f
switch(v) {
//case Packet::VERB_NOP:
default: // ignore unknown verbs, but if they pass auth check they are "received"
+ Metrics::pkt_nop_in++;
peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,false,0,ZT_QOS_NO_FLOW);
break;
case Packet::VERB_HELLO: r = _doHELLO(RR,tPtr,true); break;
@@ -131,7 +132,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
uint64_t networkId = 0;
- Metrics::pkt_error++;
+ Metrics::pkt_error_in++;
/* Security note: we do not gate doERROR() with expectingReplyTo() to
* avoid having to log every outgoing packet ID. Instead we put the
@@ -148,7 +149,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
if ((network)&&(network->controller() == peer->address()))
network->setNotFound(tPtr);
}
- Metrics::pkt_error_obj_not_found++;
+ Metrics::pkt_error_obj_not_found_in++;
break;
case Packet::ERROR_UNSUPPORTED_OPERATION:
@@ -160,7 +161,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
if ((network)&&(network->controller() == peer->address()))
network->setNotFound(tPtr);
}
- Metrics::pkt_error_unsupported_op++;
+ Metrics::pkt_error_unsupported_op_in++;
break;
case Packet::ERROR_IDENTITY_COLLISION:
@@ -168,7 +169,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
if (RR->topology->isUpstream(peer->identity())) {
RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
}
- Metrics::pkt_error_identity_collision++;
+ Metrics::pkt_error_identity_collision_in++;
break;
case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
@@ -179,7 +180,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
if ((network)&&(network->config().com)) {
network->peerRequestedCredentials(tPtr,peer->address(),now);
}
- Metrics::pkt_error_need_membership_cert++;
+ Metrics::pkt_error_need_membership_cert_in++;
} break;
case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
@@ -188,7 +189,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
if ((network)&&(network->controller() == peer->address())) {
network->setAccessDenied(tPtr);
}
- Metrics::pkt_error_network_access_denied++;
+ Metrics::pkt_error_network_access_denied_in++;
} break;
case Packet::ERROR_UNWANTED_MULTICAST: {
@@ -200,7 +201,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at<uint32_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14));
RR->mc->remove(network->id(),mg,peer->address());
}
- Metrics::pkt_error_unwanted_multicast++;
+ Metrics::pkt_error_unwanted_multicast_in++;
} break;
case Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED: {
@@ -259,7 +260,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
network->setAuthenticationRequired(tPtr, "");
}
}
- Metrics::pkt_error_authentication_required++;
+ Metrics::pkt_error_authentication_required_in++;
} break;
default: break;
@@ -286,13 +287,13 @@ bool IncomingPacket::_doACK(const RuntimeEnvironment* RR, void* tPtr, const Shar
bond->receivedAck(_path, RR->node->now(), Utils::ntoh(ackedBytes));
}
*/
- Metrics::pkt_ack++;
+ Metrics::pkt_ack_in++;
return true;
}
bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr, const SharedPtr<Peer>& peer)
{
- Metrics::pkt_qos++;
+ Metrics::pkt_qos_in++;
SharedPtr<Bond> bond = peer->bond();
if (! bond || ! bond->rateGateQoS(RR->node->now(), _path)) {
return true;
@@ -323,7 +324,7 @@ bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr
bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated)
{
- Metrics::pkt_hello++;
+ Metrics::pkt_hello_in++;
const int64_t now = RR->node->now();
const uint64_t pid = packetId();
@@ -526,7 +527,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_ok++;
+ Metrics::pkt_ok_in++;
const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
const uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID);
uint64_t networkId = 0;
@@ -644,7 +645,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar
return true;
}
- Metrics::pkt_whois++;
+ Metrics::pkt_whois_in++;
Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
outp.append((unsigned char)Packet::VERB_WHOIS);
@@ -678,7 +679,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar
bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_rendezvous++;
+ Metrics::pkt_rendezvous_in++;
if (RR->topology->isUpstream(peer->identity())) {
const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
const SharedPtr<Peer> rendezvousWith(RR->topology->getPeer(tPtr,with));
@@ -732,7 +733,7 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig
bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer,int32_t flowId)
{
- Metrics::pkt_frame++;
+ Metrics::pkt_frame_in++;
int32_t _flowId = ZT_QOS_NO_FLOW;
SharedPtr<Bond> bond = peer->bond();
if (bond && bond->flowHashingSupported()) {
@@ -825,7 +826,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer,int32_t flowId)
{
- Metrics::pkt_ext_frame++;
+ Metrics::pkt_ext_frame_in++;
const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID);
const SharedPtr<Network> network(RR->node->network(nwid));
if (network) {
@@ -908,7 +909,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_echo++;
+ Metrics::pkt_echo_in++;
uint64_t now = RR->node->now();
if (!_path->rateGateEchoRequest(now)) {
return true;
@@ -931,7 +932,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share
bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_multicast_like++;
+ Metrics::pkt_multicast_like_in++;
const int64_t now = RR->node->now();
bool authorized = false;
uint64_t lastNwid = 0;
@@ -957,7 +958,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,c
bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_network_credentials++;
+ Metrics::pkt_network_credentials_in++;
if (!peer->rateGateCredentialsReceived(RR->node->now())) {
return true;
}
@@ -1082,7 +1083,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_network_config_request++;
+ Metrics::pkt_network_config_request_in++;
const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID);
const unsigned int hopCount = hops();
const uint64_t requestPacketId = packetId();
@@ -1109,7 +1110,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void
bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_network_config++;
+ Metrics::pkt_network_config_in++;
const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PACKET_IDX_PAYLOAD)));
if (network) {
const uint64_t configUpdateId = network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD);
@@ -1133,7 +1134,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,c
bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_multicast_gather++;
+ Metrics::pkt_multicast_gather_in++;
const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID);
const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS];
const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));
@@ -1174,7 +1175,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr
bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_multicast_frame++;
+ Metrics::pkt_multicast_frame_in++;
const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
@@ -1275,7 +1276,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_push_direct_paths++;
+ Metrics::pkt_push_direct_paths_in++;
const int64_t now = RR->node->now();
if (!peer->rateGatePushDirectPaths(now)) {
@@ -1338,7 +1339,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_user_message++;
+ Metrics::pkt_user_message_in++;
if (likely(size() >= (ZT_PACKET_IDX_PAYLOAD + 8))) {
ZT_UserMessage um;
um.origin = peer->address().toInt();
@@ -1355,7 +1356,7 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con
bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_remote_trace++;
+ Metrics::pkt_remote_trace_in++;
ZT_RemoteTrace rt;
const char *ptr = reinterpret_cast<const char *>(data()) + ZT_PACKET_IDX_PAYLOAD;
const char *const eof = reinterpret_cast<const char *>(data()) + size();
@@ -1380,7 +1381,7 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con
bool IncomingPacket::_doPATH_NEGOTIATION_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
{
- Metrics::pkt_path_negotiation_request++;
+ Metrics::pkt_path_negotiation_request_in++;
uint64_t now = RR->node->now();
SharedPtr<Bond> bond = peer->bond();
if (!bond || !bond->rateGatePathNegotiation(now, _path)) {
diff --git a/node/Metrics.cpp b/node/Metrics.cpp
index 8ccef869f..d57bb2d00 100644
--- a/node/Metrics.cpp
+++ b/node/Metrics.cpp
@@ -25,64 +25,135 @@ namespace ZeroTier {
// Packet Type Counts
prometheus::simpleapi::counter_family_t packets
{ "zt_packet_incoming", "incoming packet type counts"};
- prometheus::simpleapi::counter_metric_t pkt_error
- { packets.Add({{"packet_type", "error"}}) };
- prometheus::simpleapi::counter_metric_t pkt_ack
- { packets.Add({{"packet_type", "ack"}}) };
- prometheus::simpleapi::counter_metric_t pkt_qos
- { packets.Add({{"packet_type", "qos"}}) };
- prometheus::simpleapi::counter_metric_t pkt_hello
- { packets.Add({{"packet_type", "hello"}}) };
- prometheus::simpleapi::counter_metric_t pkt_ok
- { packets.Add({{"packet_type", "ok"}}) };
- prometheus::simpleapi::counter_metric_t pkt_whois
- { packets.Add({{"packet_type", "whois"}}) };
- prometheus::simpleapi::counter_metric_t pkt_rendezvous
- { packets.Add({{"packet_type", "rendezvous"}}) };
- prometheus::simpleapi::counter_metric_t pkt_frame
- { packets.Add({{"packet_type", "frame"}}) };
- prometheus::simpleapi::counter_metric_t pkt_ext_frame
- { packets.Add({{"packet_type", "ext_frame"}}) };
- prometheus::simpleapi::counter_metric_t pkt_echo
- { packets.Add({{"packet_type", "echo"}}) };
- prometheus::simpleapi::counter_metric_t pkt_multicast_like
- { packets.Add({{"packet_type", "multicast_like"}}) };
- prometheus::simpleapi::counter_metric_t pkt_network_credentials
- { packets.Add({{"packet_type", "network_credentials"}}) };
- prometheus::simpleapi::counter_metric_t pkt_network_config_request
- { packets.Add({{"packet_type", "network_config_request"}}) };
- prometheus::simpleapi::counter_metric_t pkt_network_config
- { packets.Add({{"packet_type", "network_config"}}) };
- prometheus::simpleapi::counter_metric_t pkt_multicast_gather
- { packets.Add({{"packet_type", "multicast_gather"}}) };
- prometheus::simpleapi::counter_metric_t pkt_multicast_frame
- { packets.Add({{"packet_type", "multicast_frame"}}) };
- prometheus::simpleapi::counter_metric_t pkt_push_direct_paths
- { packets.Add({{"packet_type", "push_direct_paths"}}) };
- prometheus::simpleapi::counter_metric_t pkt_user_message
- { packets.Add({{"packet_type", "user_message"}}) };
- prometheus::simpleapi::counter_metric_t pkt_remote_trace
- { packets.Add({{"packet_type", "remote_trace"}}) };
- prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request
- { packets.Add({{"packet_type", "path_negotiation_request"}}) };
+
+ // Incoming packets
+ prometheus::simpleapi::counter_metric_t pkt_nop_in
+ { packets.Add({{"packet_type", "nop"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_in
+ { packets.Add({{"packet_type", "error"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ack_in
+ { packets.Add({{"packet_type", "ack"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_qos_in
+ { packets.Add({{"packet_type", "qos"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_hello_in
+ { packets.Add({{"packet_type", "hello"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ok_in
+ { packets.Add({{"packet_type", "ok"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_whois_in
+ { packets.Add({{"packet_type", "whois"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_rendezvous_in
+ { packets.Add({{"packet_type", "rendezvous"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_frame_in
+ { packets.Add({{"packet_type", "frame"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ext_frame_in
+ { packets.Add({{"packet_type", "ext_frame"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_echo_in
+ { packets.Add({{"packet_type", "echo"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_like_in
+ { packets.Add({{"packet_type", "multicast_like"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_credentials_in
+ { packets.Add({{"packet_type", "network_credentials"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_config_request_in
+ { packets.Add({{"packet_type", "network_config_request"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_config_in
+ { packets.Add({{"packet_type", "network_config"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in
+ { packets.Add({{"packet_type", "multicast_gather"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in
+ { packets.Add({{"packet_type", "multicast_frame"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in
+ { packets.Add({{"packet_type", "push_direct_paths"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_user_message_in
+ { packets.Add({{"packet_type", "user_message"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_remote_trace_in
+ { packets.Add({{"packet_type", "remote_trace"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in
+ { packets.Add({{"packet_type", "path_negotiation_request"}, {"direction", "rx"}}) };
+
+ // Outgoing packets
+ prometheus::simpleapi::counter_metric_t pkt_nop_out
+ { packets.Add({{"packet_type", "nop"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_out
+ { packets.Add({{"packet_type", "error"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ack_out
+ { packets.Add({{"packet_type", "ack"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_qos_out
+ { packets.Add({{"packet_type", "qos"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_hello_out
+ { packets.Add({{"packet_type", "hello"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ok_out
+ { packets.Add({{"packet_type", "ok"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_whois_out
+ { packets.Add({{"packet_type", "whois"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_rendezvous_out
+ { packets.Add({{"packet_type", "rendezvous"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_frame_out
+ { packets.Add({{"packet_type", "frame"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_ext_frame_out
+ { packets.Add({{"packet_type", "ext_frame"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_echo_out
+ { packets.Add({{"packet_type", "echo"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_like_out
+ { packets.Add({{"packet_type", "multicast_like"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_credentials_out
+ { packets.Add({{"packet_type", "network_credentials"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_config_request_out
+ { packets.Add({{"packet_type", "network_config_request"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_network_config_out
+ { packets.Add({{"packet_type", "network_config"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out
+ { packets.Add({{"packet_type", "multicast_gather"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out
+ { packets.Add({{"packet_type", "multicast_frame"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out
+ { packets.Add({{"packet_type", "push_direct_paths"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_user_message_out
+ { packets.Add({{"packet_type", "user_message"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_remote_trace_out
+ { packets.Add({{"packet_type", "remote_trace"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out
+ { packets.Add({{"packet_type", "path_negotiation_request"}, {"direction", "tx"}}) };
+
// Packet Error Counts
prometheus::simpleapi::counter_family_t packet_errors
{ "zt_packet_incoming_error", "incoming packet errors"};
- prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found
- { packet_errors.Add({{"error_type", "obj_not_found"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op
- { packet_errors.Add({{"error_type", "unsupported_operation"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_identity_collision
- { packet_errors.Add({{"error_type", "identity_collision"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert
- { packet_errors.Add({{"error_type", "need_membership_certificate"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied
- { packet_errors.Add({{"error_type", "network_access_denied"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast
- { packet_errors.Add({{"error_type", "unwanted_multicast"}}) };
- prometheus::simpleapi::counter_metric_t pkt_error_authentication_required
- { packet_errors.Add({{"error_type", "authentication_required"}}) };
+
+ // Incoming Error Counts
+ prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in
+ { packet_errors.Add({{"error_type", "obj_not_found"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in
+ { packet_errors.Add({{"error_type", "unsupported_operation"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in
+ { packet_errors.Add({{"error_type", "identity_collision"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in
+ { packet_errors.Add({{"error_type", "need_membership_certificate"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in
+ { packet_errors.Add({{"error_type", "network_access_denied"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in
+ { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in
+ { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "rx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in
+ { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "rx"}}) };
+
+ // Outgoing Error Counts
+ prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out
+ { packet_errors.Add({{"error_type", "obj_not_found"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out
+ { packet_errors.Add({{"error_type", "unsupported_operation"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out
+ { packet_errors.Add({{"error_type", "identity_collision"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out
+ { packet_errors.Add({{"error_type", "need_membership_certificate"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out
+ { packet_errors.Add({{"error_type", "network_access_denied"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out
+ { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out
+ { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "tx"}}) };
+ prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out
+ { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "tx"}}) };
// Data Sent/Received Metrics
prometheus::simpleapi::counter_metric_t udp_send
diff --git a/node/Metrics.hpp b/node/Metrics.hpp
index 3b65ea4b9..f38bab6b9 100644
--- a/node/Metrics.hpp
+++ b/node/Metrics.hpp
@@ -24,36 +24,75 @@ namespace ZeroTier {
namespace Metrics {
// Packet Type Counts
extern prometheus::simpleapi::counter_family_t packets;
- extern prometheus::simpleapi::counter_metric_t pkt_error;
- extern prometheus::simpleapi::counter_metric_t pkt_ack;
- extern prometheus::simpleapi::counter_metric_t pkt_qos;
- extern prometheus::simpleapi::counter_metric_t pkt_hello;
- extern prometheus::simpleapi::counter_metric_t pkt_ok;
- extern prometheus::simpleapi::counter_metric_t pkt_whois;
- extern prometheus::simpleapi::counter_metric_t pkt_rendezvous;
- extern prometheus::simpleapi::counter_metric_t pkt_frame;
- extern prometheus::simpleapi::counter_metric_t pkt_ext_frame;
- extern prometheus::simpleapi::counter_metric_t pkt_echo;
- extern prometheus::simpleapi::counter_metric_t pkt_multicast_like;
- extern prometheus::simpleapi::counter_metric_t pkt_network_credentials;
- extern prometheus::simpleapi::counter_metric_t pkt_network_config_request;
- extern prometheus::simpleapi::counter_metric_t pkt_network_config;
- extern prometheus::simpleapi::counter_metric_t pkt_multicast_gather;
- extern prometheus::simpleapi::counter_metric_t pkt_multicast_frame;
- extern prometheus::simpleapi::counter_metric_t pkt_push_direct_paths;
- extern prometheus::simpleapi::counter_metric_t pkt_user_message;
- extern prometheus::simpleapi::counter_metric_t pkt_remote_trace;
- extern prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request;
+
+ // incoming packets
+ extern prometheus::simpleapi::counter_metric_t pkt_nop_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_ack_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_qos_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_hello_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_ok_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_whois_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_rendezvous_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_frame_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_ext_frame_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_echo_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_like_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_credentials_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_config_request_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_config_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_user_message_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_remote_trace_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in;
+
+ // outgoing packets
+ extern prometheus::simpleapi::counter_metric_t pkt_nop_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_ack_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_qos_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_hello_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_ok_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_whois_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_rendezvous_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_frame_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_ext_frame_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_echo_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_like_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_credentials_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_config_request_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_network_config_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_user_message_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_remote_trace_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out;
// Packet Error Counts
extern prometheus::simpleapi::counter_family_t packet_errors;
- extern prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found;
- extern prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op;
- extern prometheus::simpleapi::counter_metric_t pkt_error_identity_collision;
- extern prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert;
- extern prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied;
- extern prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast;
- extern prometheus::simpleapi::counter_metric_t pkt_error_authentication_required;
+
+ // incoming errors
+ extern prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in;
+
+ // outgoing errors
+ extern prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out;
+ extern prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out;
// Data Sent/Received Metrics
diff --git a/node/Node.cpp b/node/Node.cpp
index 46f3a4add..a32782edb 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -34,6 +34,7 @@
#include "SelfAwareness.hpp"
#include "Network.hpp"
#include "Trace.hpp"
+#include "Metrics.hpp"
namespace ZeroTier {
@@ -769,7 +770,6 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
break;
case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED: {
//fprintf(stderr, "\n\nGot auth required\n\n");
-
break;
}
@@ -784,12 +784,15 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
//case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR:
default:
outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
+ Metrics::pkt_error_obj_not_found_out++;
break;
case NetworkController::NC_ERROR_ACCESS_DENIED:
outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
+ Metrics::pkt_error_network_access_denied_out++;
break;
case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED:
outp.append((unsigned char)Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED);
+ Metrics::pkt_error_authentication_required_out++;
break;
}
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 5cb55fa4b..34e3b9c4f 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -22,6 +22,7 @@
#include "InetAddress.hpp"
#include "RingBuffer.hpp"
#include "Utils.hpp"
+#include "Metrics.hpp"
namespace ZeroTier {
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 59ed34a04..7b1d47ac9 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -31,6 +31,7 @@
#include "SelfAwareness.hpp"
#include "Packet.hpp"
#include "Trace.hpp"
+#include "Metrics.hpp"
namespace ZeroTier {
@@ -850,8 +851,10 @@ void Switch::removeNetworkQoSControlBlock(uint64_t nwid)
void Switch::send(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
{
const Address dest(packet.destination());
- if (dest == RR->identity.address())
+ if (dest == RR->identity.address()) {
return;
+ }
+ _recordOutgoingPacketMetrics(packet);
if (!_trySend(tPtr,packet,encrypt,flowId)) {
{
Mutex::Lock _l(_txQueue_m);
@@ -1080,4 +1083,72 @@ void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path
}
}
+void Switch::_recordOutgoingPacketMetrics(const Packet &p) {
+ switch (p.verb()) {
+ case Packet::VERB_NOP:
+ Metrics::pkt_nop_out++;
+ break;
+ case Packet::VERB_HELLO:
+ Metrics::pkt_hello_out++;
+ break;
+ case Packet::VERB_ERROR:
+ Metrics::pkt_error_out++;
+ break;
+ case Packet::VERB_OK:
+ Metrics::pkt_ok_out++;
+ break;
+ case Packet::VERB_WHOIS:
+ Metrics::pkt_whois_out++;
+ break;
+ case Packet::VERB_RENDEZVOUS:
+ Metrics::pkt_rendezvous_out++;
+ break;
+ case Packet::VERB_FRAME:
+ Metrics::pkt_frame_out++;
+ break;
+ case Packet::VERB_EXT_FRAME:
+ Metrics::pkt_ext_frame_out++;
+ break;
+ case Packet::VERB_ECHO:
+ Metrics::pkt_echo_out++;
+ break;
+ case Packet::VERB_MULTICAST_LIKE:
+ Metrics::pkt_multicast_like_out++;
+ break;
+ case Packet::VERB_NETWORK_CREDENTIALS:
+ Metrics::pkt_network_credentials_out++;
+ break;
+ case Packet::VERB_NETWORK_CONFIG_REQUEST:
+ Metrics::pkt_network_config_request_out++;
+ break;
+ case Packet::VERB_NETWORK_CONFIG:
+ Metrics::pkt_network_config_out++;
+ break;
+ case Packet::VERB_MULTICAST_GATHER:
+ Metrics::pkt_multicast_gather_out++;
+ break;
+ case Packet::VERB_MULTICAST_FRAME:
+ Metrics::pkt_multicast_frame_out++;
+ break;
+ case Packet::VERB_PUSH_DIRECT_PATHS:
+ Metrics::pkt_push_direct_paths_out++;
+ break;
+ case Packet::VERB_ACK:
+ Metrics::pkt_ack_out++;
+ break;
+ case Packet::VERB_QOS_MEASUREMENT:
+ Metrics::pkt_qos_out++;
+ break;
+ case Packet::VERB_USER_MESSAGE:
+ Metrics::pkt_user_message_out++;
+ break;
+ case Packet::VERB_REMOTE_TRACE:
+ Metrics::pkt_remote_trace_out++;
+ break;
+ case Packet::VERB_PATH_NEGOTIATION_REQUEST:
+ Metrics::pkt_path_negotiation_request_out++;
+ break;
+ }
+}
+
} // namespace ZeroTier
diff --git a/node/Switch.hpp b/node/Switch.hpp
index 9bb1fbeaf..b1f1bf0ce 100644
--- a/node/Switch.hpp
+++ b/node/Switch.hpp
@@ -208,6 +208,7 @@ private:
bool _shouldUnite(const int64_t now,const Address &source,const Address &destination);
bool _trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId = ZT_QOS_NO_FLOW); // packet is modified if return is true
void _sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId);
+ void _recordOutgoingPacketMetrics(const Packet &p);
const RuntimeEnvironment *const RR;
int64_t _lastBeaconResponse;