This is an automated email from the ASF dual-hosted git repository. cmcfarlen pushed a commit to branch 10.2.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit c28a230947399a74e0a1100e98865be049aedc0e Author: Masakazu Kitajo <[email protected]> AuthorDate: Fri Apr 10 09:33:27 2026 -0600 Refactor jax_fingerprint plugin for better modularity and fewer allocations (#13072) Each subdirectory (ja3/, ja4/, ja4h/) now follows a consistent structure: method.cc/.h for the fingerprint method, test.cc for tests, plus algorithm-specific files. Redundant prefixes removed since subdirectories provide context. Tests updated to match. No functional changes. (cherry picked from commit ad2940c776ee86d9eac24a2ee4dc4d80c91e9991) --- .../experimental/jax_fingerprint/CMakeLists.txt | 24 +- .../ja3/{ja3_method.cc => method.cc} | 36 +- .../{ja4/ja4_method.h => ja3/method.h} | 2 +- .../jax_fingerprint/ja3/{test_ja3.cc => test.cc} | 2 +- .../jax_fingerprint/ja3/{ja3_utils.cc => utils.cc} | 0 .../jax_fingerprint/ja3/{ja3_utils.h => utils.h} | 0 .../experimental/jax_fingerprint/ja4/datasource.cc | 73 +++ .../experimental/jax_fingerprint/ja4/datasource.h | 72 +++ plugins/experimental/jax_fingerprint/ja4/ja4.cc | 236 +++++---- plugins/experimental/jax_fingerprint/ja4/ja4.h | 144 +----- .../experimental/jax_fingerprint/ja4/ja4_method.cc | 159 ------ .../jax_fingerprint/{ja4h/ja4h.h => ja4/method.cc} | 51 +- .../{ja3/ja3_method.h => ja4/method.h} | 2 +- plugins/experimental/jax_fingerprint/ja4/test.cc | 532 +++++++++++++++++++++ .../experimental/jax_fingerprint/ja4/test_ja4.cc | 425 ---------------- .../ja4/tls_client_hello_summary.cc | 215 ++++++--- .../jax_fingerprint/ja4/tls_client_hello_summary.h | 55 +++ .../{ja4h/ja4h_method.h => ja4_common/utils.cc} | 17 +- .../{ja4/ja4_method.h => ja4_common/utils.h} | 9 +- plugins/experimental/jax_fingerprint/ja4h/ja4h.cc | 2 +- plugins/experimental/jax_fingerprint/ja4h/ja4h.h | 7 +- .../ja4h/{ja4h_method.cc => method.cc} | 30 +- .../ja4h/{ja4h_method.h => method.h} | 2 +- .../jax_fingerprint/ja4h/{test_ja4h.cc => test.cc} | 47 +- plugins/experimental/jax_fingerprint/plugin.cc | 17 +- 25 files changed, 1178 insertions(+), 981 deletions(-) diff --git a/plugins/experimental/jax_fingerprint/CMakeLists.txt b/plugins/experimental/jax_fingerprint/CMakeLists.txt index 173eb5750b..cdcc587390 100644 --- a/plugins/experimental/jax_fingerprint/CMakeLists.txt +++ b/plugins/experimental/jax_fingerprint/CMakeLists.txt @@ -22,30 +22,34 @@ add_atsplugin( userarg.cc header.cc log.cc - ja3/ja3_method.cc - ja3/ja3_utils.cc - ja4/ja4_method.cc + ja3/method.cc + ja3/utils.cc + ja4/method.cc ja4/ja4.cc + ja4/datasource.cc ja4/tls_client_hello_summary.cc - ja4h/ja4h_method.cc + ja4h/method.cc ja4h/ja4h.cc ja4h/datasource.cc + ja4_common/utils.cc ) target_link_libraries(jax_fingerprint PRIVATE OpenSSL::Crypto OpenSSL::SSL) -verify_global_plugin(jax_fingerprint) target_include_directories(jax_fingerprint BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +verify_global_plugin(jax_fingerprint) +verify_remap_plugin(jax_fingerprint) if(BUILD_TESTING) add_executable( test_jax - ja3/test_ja3.cc - ja3/ja3_utils.cc - ja4/test_ja4.cc + ja3/test.cc + ja3/utils.cc + ja4/test.cc ja4/ja4.cc - ja4/tls_client_hello_summary.cc - ja4h/test_ja4h.cc + ja4/datasource.cc + ja4h/test.cc ja4h/ja4h.cc ja4h/datasource.cc + ja4_common/utils.cc ) target_link_libraries(test_jax PRIVATE Catch2::Catch2WithMain OpenSSL::Crypto OpenSSL::SSL) target_include_directories(test_jax BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/plugins/experimental/jax_fingerprint/ja3/ja3_method.cc b/plugins/experimental/jax_fingerprint/ja3/method.cc similarity index 94% rename from plugins/experimental/jax_fingerprint/ja3/ja3_method.cc rename to plugins/experimental/jax_fingerprint/ja3/method.cc index ef2e0a2160..4d8beae5d0 100644 --- a/plugins/experimental/jax_fingerprint/ja3/ja3_method.cc +++ b/plugins/experimental/jax_fingerprint/ja3/method.cc @@ -24,8 +24,8 @@ #include <plugin.h> #include <context.h> -#include "ja3_method.h" -#include "ja3_utils.h" +#include "method.h" +#include "utils.h" #include <openssl/ssl.h> #include <openssl/md5.h> @@ -33,20 +33,6 @@ #include <algorithm> -namespace ja3_method -{ - -void on_client_hello(JAxContext *, TSVConn); - -struct Method method = { - "JA3", - Method::Type::CONNECTION_BASED, - on_client_hello, - nullptr, -}; - -} // namespace ja3_method - namespace { constexpr int ja3_hash_included_byte_count{16}; @@ -54,8 +40,6 @@ static_assert(ja3_hash_included_byte_count <= MD5_DIGEST_LENGTH); constexpr int ja3_hash_hex_string_with_null_terminator_length{2 * ja3_hash_included_byte_count + 1}; -} // end anonymous namespace - static std::string get_fingerprint(TSClientHello ch) { @@ -115,8 +99,13 @@ get_fingerprint(TSClientHello ch) return {fingerprint}; } +} // end anonymous namespace + +namespace ja3 +{ + void -ja3_method::on_client_hello(JAxContext *ctx, TSVConn vconn) +on_client_hello(JAxContext *ctx, TSVConn vconn) { TSClientHello ch = TSVConnClientHelloGet(vconn); @@ -126,3 +115,12 @@ ja3_method::on_client_hello(JAxContext *ctx, TSVConn vconn) ctx->set_fingerprint(get_fingerprint(ch)); } } + +struct Method method = { + "JA3", + Method::Type::CONNECTION_BASED, + on_client_hello, + nullptr, +}; + +} // namespace ja3 diff --git a/plugins/experimental/jax_fingerprint/ja4/ja4_method.h b/plugins/experimental/jax_fingerprint/ja3/method.h similarity index 97% copy from plugins/experimental/jax_fingerprint/ja4/ja4_method.h copy to plugins/experimental/jax_fingerprint/ja3/method.h index 4b7abea288..9f6a51ef57 100644 --- a/plugins/experimental/jax_fingerprint/ja4/ja4_method.h +++ b/plugins/experimental/jax_fingerprint/ja3/method.h @@ -24,7 +24,7 @@ #include <method.h> -namespace ja4_method +namespace ja3 { extern struct Method method; diff --git a/plugins/experimental/jax_fingerprint/ja3/test_ja3.cc b/plugins/experimental/jax_fingerprint/ja3/test.cc similarity index 98% rename from plugins/experimental/jax_fingerprint/ja3/test_ja3.cc rename to plugins/experimental/jax_fingerprint/ja3/test.cc index 1e5de5e25d..9826ef2405 100644 --- a/plugins/experimental/jax_fingerprint/ja3/test_ja3.cc +++ b/plugins/experimental/jax_fingerprint/ja3/test.cc @@ -22,7 +22,7 @@ */ -#include "ja3_utils.h" +#include "utils.h" #include <catch2/catch_test_macros.hpp> diff --git a/plugins/experimental/jax_fingerprint/ja3/ja3_utils.cc b/plugins/experimental/jax_fingerprint/ja3/utils.cc similarity index 100% rename from plugins/experimental/jax_fingerprint/ja3/ja3_utils.cc rename to plugins/experimental/jax_fingerprint/ja3/utils.cc diff --git a/plugins/experimental/jax_fingerprint/ja3/ja3_utils.h b/plugins/experimental/jax_fingerprint/ja3/utils.h similarity index 100% rename from plugins/experimental/jax_fingerprint/ja3/ja3_utils.h rename to plugins/experimental/jax_fingerprint/ja3/utils.h diff --git a/plugins/experimental/jax_fingerprint/ja4/datasource.cc b/plugins/experimental/jax_fingerprint/ja4/datasource.cc new file mode 100644 index 0000000000..9ed2fa3757 --- /dev/null +++ b/plugins/experimental/jax_fingerprint/ja4/datasource.cc @@ -0,0 +1,73 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "datasource.h" + +#include <array> +#include <algorithm> + +constexpr std::array<std::uint16_t, 16> GREASE_values{0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, + 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba, 0xcaca, 0xdada, 0xeaea, 0xfafa}; + +ja4::Datasource::Protocol +ja4::Datasource::get_protocol() +{ + return this->_protocol; +} + +int +ja4::Datasource::get_version() +{ + return this->_version; +} + +ja4::Datasource::SNI +ja4::Datasource::get_sni_type() +{ + return this->_has_SNI ? ja4::Datasource::SNI::to_domain : ja4::Datasource::SNI::to_IP; +} + +int +ja4::Datasource::get_cipher_count() +{ + return this->_n_ciphers; +} + +int +ja4::Datasource::get_extension_count() +{ + return this->_n_extensions + (this->_has_ALPN ? 1 : 0) + (this->_has_SNI ? 1 : 0); +} + +/** + * Check whether @a value is a GREASE value. + * + * These are reserved extensions randomly advertised to keep implementations + * well lubricated. They are ignored in all parts of JA4 because of their + * random nature. + * + * @return Returns true if the value is a GREASE value, false otherwise. + */ +bool +ja4::Datasource::_is_GREASE(uint16_t value) +{ + return std::binary_search(GREASE_values.begin(), GREASE_values.end(), value); +} diff --git a/plugins/experimental/jax_fingerprint/ja4/datasource.h b/plugins/experimental/jax_fingerprint/ja4/datasource.h new file mode 100644 index 0000000000..7d484b39be --- /dev/null +++ b/plugins/experimental/jax_fingerprint/ja4/datasource.h @@ -0,0 +1,72 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#pragma once + +#include <string_view> +#include <cstdint> + +constexpr uint16_t EXT_SNI{0x0}; +constexpr uint16_t EXT_ALPN{0x10}; +constexpr uint16_t EXT_SUPPORTED_VERSIONS{0x2b}; + +namespace ja4 +{ + +class Datasource +{ +public: + Datasource() = default; + virtual ~Datasource() = default; + + enum class Protocol { + DTLS = 'd', + QUIC = 'q', + TLS = 't', + }; + + enum class SNI { + to_domain = 'd', + to_IP = 'i', + }; + + Protocol get_protocol(); + int get_version(); + SNI get_sni_type(); + int get_cipher_count(); + int get_extension_count(); + virtual std::string_view get_first_alpn() = 0; + virtual void get_cipher_suites_hash(unsigned char out[32]) = 0; + virtual void get_extension_hash(unsigned char out[32]) = 0; + +protected: + bool _is_GREASE(uint16_t value); + + Protocol _protocol; + int _version = 0; + bool _has_ALPN = false; + bool _has_SNI = false; + int _n_ciphers = 0; + int _n_extensions = 0; +}; + +} // namespace ja4 diff --git a/plugins/experimental/jax_fingerprint/ja4/ja4.cc b/plugins/experimental/jax_fingerprint/ja4/ja4.cc index 62a6c08fa4..48e790b2fa 100644 --- a/plugins/experimental/jax_fingerprint/ja4/ja4.cc +++ b/plugins/experimental/jax_fingerprint/ja4/ja4.cc @@ -22,6 +22,9 @@ */ #include "ja4.h" +#include <ja4_common/utils.h> + +#include <openssl/sha.h> #include <algorithm> #include <cctype> @@ -29,147 +32,188 @@ #include <cstdint> #include <cstdio> #include <iterator> +#include <vector> #include <string> #include <string_view> -static char convert_protocol_to_char(JA4::Protocol protocol); -static std::string convert_TLS_version_to_string(std::uint16_t TLS_version); -static char convert_SNI_to_char(JA4::SNI SNI_type); -static std::string convert_count_to_two_digit_string(std::size_t count); -static std::string convert_ALPN_to_two_char_string(std::string_view ALPN); -static void remove_trailing_character(std::string &s); -static std::string hexify(std::uint16_t n); +static void convert_protocol_to_char(char *out, ja4::Datasource::Protocol protocol); +static void convert_TLS_version_to_string(char *out, std::uint16_t TLS_version); +static void convert_SNI_to_char(char *out, ja4::Datasource::SNI SNI_type); +static void convert_count_to_two_digit_string(char *out, std::size_t count); +static void convert_ALPN_to_two_char_string(char *out, std::string_view ALPN); -namespace +/** + * Calculate the a portion of the JA4 fingerprint for the given client hello. + * + * The a portion of the fingerprint encodes the protocol, TLS version, SNI + * type, number of cipher suites, number of extensions, and first ALPN value. + * + * For more information see: + * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. + * + * @param TLS_summary The TLS client hello. + * @return Returns a string containing the a portion of the JA4 fingerprint. + */ +static void +make_JA4_a(char *out, ja4::Datasource &datasource) { -constexpr std::size_t U16_HEX_BUF_SIZE{4}; -} // end anonymous namespace + convert_protocol_to_char(out, datasource.get_protocol()); + out += 1; -std::string -JA4::make_JA4_a_raw(TLSClientHelloSummary const &TLS_summary) -{ - std::string result; - result.reserve(9); - result.push_back(convert_protocol_to_char(TLS_summary.protocol)); - result.append(convert_TLS_version_to_string(TLS_summary.TLS_version)); - result.push_back(convert_SNI_to_char(TLS_summary.get_SNI_type())); - result.append(convert_count_to_two_digit_string(TLS_summary.get_cipher_count())); - result.append(convert_count_to_two_digit_string(TLS_summary.get_extension_count())); - result.append(convert_ALPN_to_two_char_string(TLS_summary.ALPN)); - return result; + convert_TLS_version_to_string(out, datasource.get_version()); + out += 2; + + convert_SNI_to_char(out, datasource.get_sni_type()); + out += 1; + + convert_count_to_two_digit_string(out, datasource.get_cipher_count()); + out += 2; + + convert_count_to_two_digit_string(out, datasource.get_extension_count()); + out += 2; + + convert_ALPN_to_two_char_string(out, datasource.get_first_alpn()); } -static char -convert_protocol_to_char(JA4::Protocol protocol) +static void +convert_protocol_to_char(char *out, ja4::Datasource::Protocol protocol) { - return static_cast<char>(protocol); + out[0] = static_cast<char>(protocol); } -static std::string -convert_TLS_version_to_string(std::uint16_t TLS_version) +static void +convert_TLS_version_to_string(char *out, std::uint16_t version) { - switch (TLS_version) { + switch (version) { case 0x304: - return "13"; + out[0] = '1'; + out[1] = '3'; + break; case 0x303: - return "12"; + out[0] = '1'; + out[1] = '2'; + break; case 0x302: - return "11"; + out[0] = '1'; + out[1] = '1'; + break; case 0x301: - return "10"; + out[0] = '1'; + out[1] = '0'; + break; case 0x300: - return "s3"; + out[0] = 's'; + out[1] = '3'; + break; case 0x200: - return "s2"; + out[0] = 's'; + out[1] = '2'; + break; case 0x100: - return "s1"; + out[0] = 's'; + out[1] = '1'; + break; case 0xfeff: - return "d1"; + out[0] = 'd'; + out[1] = '1'; + break; case 0xfefd: - return "d2"; + out[0] = 'd'; + out[1] = '2'; + break; case 0xfefc: - return "d3"; + out[0] = 'd'; + out[1] = '3'; + break; default: - return "00"; + out[0] = '0'; + out[1] = '0'; + break; } } -static char -convert_SNI_to_char(JA4::SNI SNI_type) +static void +convert_SNI_to_char(char *out, ja4::Datasource::SNI type) { - return static_cast<char>(SNI_type); + out[0] = static_cast<char>(type); } -static std::string -convert_count_to_two_digit_string(std::size_t count) +static void +convert_count_to_two_digit_string(char *out, std::size_t count) { - std::string result; - if (count <= 9) { - result.push_back('0'); + if (count <= 99) { + out[0] = (count / 10) + '0'; + out[1] = (count % 10) + '0'; + } else { + out[0] = '9'; + out[1] = '9'; } - // We could also clamp the lower bound to 1 since there must be at least 1 - // cipher, but 0 is more helpful for debugging if the cipher list is empty. - result.append(std::to_string(std::clamp(count, std::size_t{0}, std::size_t{99}))); - return result; } -static std::string -convert_ALPN_to_two_char_string(std::string_view ALPN) +static void +convert_ALPN_to_two_char_string(char *out, std::string_view alpn) { - std::string result; - if (ALPN.empty()) { - result = "00"; + if (alpn.empty()) { + out[0] = '0'; + out[1] = '0'; } else { - result.push_back(ALPN.front()); - result.push_back(ALPN.back()); + out[0] = alpn.front(); + out[1] = alpn.back(); } - return result; } -std::string -JA4::make_JA4_b_raw(TLSClientHelloSummary const &TLS_summary) +/** + * Calculate the b portion of the JA4 fingerprint for the given client hello. + * + * The b portion of the fingerprint is a comma-delimited list of lowercase hex + * numbers representing the cipher suites in sorted order. GREASE values are + * ignored. + * + * For more information see: + * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. + * + * @param TLS_summary The TLS client hello. + * @return Returns a string containing the b portion of the JA4 fingerprint. + */ +static void +make_JA4_b(char *out, ja4::Datasource &datasource) { - std::string result; - result.reserve(12); - std::vector temp = TLS_summary.get_ciphers(); - std::sort(temp.begin(), temp.end()); - - for (auto cipher : temp) { - result.append(hexify(cipher)); - result.push_back(','); - } - remove_trailing_character(result); - return result; -} + unsigned char hash[32]; -std::string -JA4::make_JA4_c_raw(TLSClientHelloSummary const &TLS_summary) -{ - std::string result; - result.reserve(12); - std::vector temp = TLS_summary.get_extensions(); - std::sort(temp.begin(), temp.end()); - - for (auto extension : temp) { - result.append(hexify(extension)); - result.push_back(','); - } - remove_trailing_character(result); - return result; + datasource.get_cipher_suites_hash(hash); + hash_stringify(out, hash); } +/** + * Calculate the c portion of the JA4 fingerprint for the given client hello. + * + * The b portion of the fingerprint is a comma-delimited list of lowercase hex + * numbers representing the extensions in sorted order. GREASE values and the + * SNI and ALPN extensions are ignored. + * + * For more information see: + * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. + * + * @param TLS_summary The TLS client hello. + * @return Returns a string containing the c portion of the JA4 fingerprint. + */ static void -remove_trailing_character(std::string &s) +make_JA4_c(char *out, ja4::Datasource &datasource) { - if (!s.empty()) { - s.pop_back(); - } + unsigned char hash[32]; + + datasource.get_extension_hash(hash); + hash_stringify(out, hash); } -static std::string -hexify(std::uint16_t n) +std::string_view +ja4::generate_fingerprint(char *out, ja4::Datasource &datasource) { - char result[U16_HEX_BUF_SIZE + 1]{}; - std::snprintf(result, sizeof(result), "%.4x", n); - return result; + make_JA4_a(&(out[ja4::PART_A_POSITION]), datasource); + out[ja4::DELIMITER_1_POSITION] = ja4::PORTION_DELIMITER; + make_JA4_b(&(out[ja4::PART_B_POSITION]), datasource); + out[ja4::DELIMITER_2_POSITION] = ja4::PORTION_DELIMITER; + make_JA4_c(&(out[ja4::PART_C_POSITION]), datasource); + + return {out, FINGERPRINT_LENGTH}; } diff --git a/plugins/experimental/jax_fingerprint/ja4/ja4.h b/plugins/experimental/jax_fingerprint/ja4/ja4.h index b6be1e40cf..935223f793 100644 --- a/plugins/experimental/jax_fingerprint/ja4/ja4.h +++ b/plugins/experimental/jax_fingerprint/ja4/ja4.h @@ -22,115 +22,21 @@ #pragma once -#include <cstdint> -#include <iterator> -#include <string> -#include <vector> +#include "datasource.h" -namespace JA4 +namespace ja4 { -constexpr char PORTION_DELIMITER{'_'}; - -enum class Protocol { - DTLS = 'd', - QUIC = 'q', - TLS = 't', -}; - -enum class SNI { - to_domain = 'd', - to_IP = 'i', -}; - -/** - * Represents the data sent in a TLS Client Hello needed for JA4 fingerprints. - */ -class TLSClientHelloSummary -{ -public: - using difference_type = std::iterator_traits<std::vector<std::uint16_t>::iterator>::difference_type; - - Protocol protocol; - std::uint16_t TLS_version{0}; // 0 is not the default, this is only to not have it un-initialized. - std::string ALPN; - - std::vector<std::uint16_t> const &get_ciphers() const; - void add_cipher(std::uint16_t cipher); - - std::vector<std::uint16_t> const &get_extensions() const; - void add_extension(std::uint16_t extension); - - /** - * Get the number of ciphers excluding GREASE values. - * - * @return Returns the count of non-GREASE ciphers. - */ - difference_type get_cipher_count() const; - - /** - * Get the number of extensions excluding GREASE values. - * - * @return Returns the count of non-GREASE extensions. - */ - difference_type get_extension_count() const; - - /** Get the SNI type, domain or IP. - * - * @return Returns SNI::to_domain or SNI::to_IP. - */ - SNI get_SNI_type() const; - -private: - std::vector<std::uint16_t> _ciphers; - std::vector<std::uint16_t> _extensions; - int _extension_count_including_sni_and_alpn{0}; - SNI _SNI_type{SNI::to_IP}; -}; - -/** - * Calculate the a portion of the JA4 fingerprint for the given client hello. - * - * The a portion of the fingerprint encodes the protocol, TLS version, SNI - * type, number of cipher suites, number of extensions, and first ALPN value. - * - * For more information see: - * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. - * - * @param TLS_summary The TLS client hello. - * @return Returns a string containing the a portion of the JA4 fingerprint. - */ -std::string make_JA4_a_raw(TLSClientHelloSummary const &TLS_summary); - -/** - * Calculate the b portion of the JA4 fingerprint for the given client hello. - * - * The b portion of the fingerprint is a comma-delimited list of lowercase hex - * numbers representing the cipher suites in sorted order. GREASE values are - * ignored. - * - * For more information see: - * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. - * - * @param TLS_summary The TLS client hello. - * @return Returns a string containing the b portion of the JA4 fingerprint. - */ -std::string make_JA4_b_raw(TLSClientHelloSummary const &TLS_summary); - -/** - * Calculate the c portion of the JA4 fingerprint for the given client hello. - * - * The b portion of the fingerprint is a comma-delimited list of lowercase hex - * numbers representing the extensions in sorted order. GREASE values and the - * SNI and ALPN extensions are ignored. - * - * For more information see: - * https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md. - * - * @param TLS_summary The TLS client hello. - * @return Returns a string containing the c portion of the JA4 fingerprint. - */ -std::string make_JA4_c_raw(TLSClientHelloSummary const &TLS_summary); +constexpr int FINGERPRINT_LENGTH = 36; +constexpr size_t PART_A_POSITION = 0; +constexpr size_t PART_B_POSITION = 11; +constexpr size_t PART_C_POSITION = 24; +constexpr size_t PART_A_LENGTH = 10; +constexpr size_t PART_B_LENGTH = 12; +constexpr size_t PART_C_LENGTH = 12; +constexpr char PORTION_DELIMITER{'_'}; +constexpr size_t DELIMITER_1_POSITION = 10; +constexpr size_t DELIMITER_2_POSITION = 23; /** * Calculate the JA4 fingerprint for the given TLS client hello. @@ -144,28 +50,6 @@ std::string make_JA4_c_raw(TLSClientHelloSummary const &TLS_summary); * JA4 fingerprint, this should be a sha256 hash. * @return Returns a string containing the JA4 fingerprint. */ -template <typename UnaryOp> -std::string -make_JA4_fingerprint(TLSClientHelloSummary const &TLS_summary, UnaryOp hasher) -{ - std::string result; - result.append(make_JA4_a_raw(TLS_summary)); - result.push_back(JA4::PORTION_DELIMITER); - result.append(hasher(make_JA4_b_raw(TLS_summary)).substr(0, 12)); - result.push_back(JA4::PORTION_DELIMITER); - result.append(hasher(make_JA4_c_raw(TLS_summary)).substr(0, 12)); - return result; -} - -/** - * Check whether @a value is a GREASE value. - * - * These are reserved extensions randomly advertised to keep implementations - * well lubricated. They are ignored in all parts of JA4 because of their - * random nature. - * - * @return Returns true if the value is a GREASE value, false otherwise. - */ -bool is_GREASE(std::uint16_t value); +std::string_view generate_fingerprint(char *out, Datasource &datasource); -} // end namespace JA4 +} // end namespace ja4 diff --git a/plugins/experimental/jax_fingerprint/ja4/ja4_method.cc b/plugins/experimental/jax_fingerprint/ja4/ja4_method.cc deleted file mode 100644 index 7fcaaf2edc..0000000000 --- a/plugins/experimental/jax_fingerprint/ja4/ja4_method.cc +++ /dev/null @@ -1,159 +0,0 @@ -/** @file - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "ts/ts.h" - -#include <plugin.h> -#include <context.h> -#include "ja4_method.h" -#include "ja4.h" - -#include <openssl/sha.h> -#include <openssl/ssl.h> - -constexpr unsigned int EXT_ALPN{0x10}; -constexpr unsigned int EXT_SUPPORTED_VERSIONS{0x2b}; - -namespace ja4_method -{ - -void on_client_hello(JAxContext *, TSVConn); - -struct Method method = { - "JA4", - Method::Type::CONNECTION_BASED, - on_client_hello, - nullptr, -}; - -} // namespace ja4_method - -static std::uint16_t -get_version(TSClientHello ch) -{ - unsigned char const *buf{}; - std::size_t buflen{}; - if (TS_SUCCESS == TSClientHelloExtensionGet(ch, EXT_SUPPORTED_VERSIONS, &buf, &buflen)) { - std::uint16_t max_version{0}; - size_t versions_len = buf[0]; - - if (buflen < versions_len + 1) { - Dbg(dbg_ctl, "Malformed supported_versions extension (truncated vector)... using legacy version."); - return ch.get_version(); - } - - for (size_t i = 1; (i + 1) < (versions_len + 1); i += 2) { - std::uint16_t version = (buf[i] << 8) | buf[i + 1]; - if (!JA4::is_GREASE(version) && version > max_version) { - max_version = version; - } - } - return max_version; - } else { - Dbg(dbg_ctl, "No supported_versions extension... using legacy version."); - return ch.get_version(); - } -} - -static std::string -get_first_ALPN(TSClientHello ch) -{ - unsigned char const *buf{}; - std::size_t buflen{}; - std::string result{""}; - if (TS_SUCCESS == TSClientHelloExtensionGet(ch, EXT_ALPN, &buf, &buflen)) { - // The first two bytes are a 16bit encoding of the total length. - TSAssert(buflen > 4); - unsigned char first_ALPN_length{buf[2]}; - TSAssert(0 != first_ALPN_length); - result.assign(&buf[3], (&buf[3]) + first_ALPN_length); - } - - return result; -} - -static constexpr std::uint16_t -make_word(unsigned char lowbyte, unsigned char highbyte) -{ - return (static_cast<std::uint16_t>(highbyte) << 8) | lowbyte; -} - -static void -add_ciphers(JA4::TLSClientHelloSummary &summary, TSClientHello ch) -{ - const uint8_t *buf = ch.get_cipher_suites(); - size_t buflen = ch.get_cipher_suites_len(); - - if (buflen > 0) { - for (std::size_t i = 0; i + 1 < buflen; i += 2) { - summary.add_cipher(make_word(buf[i], buf[i + 1])); - } - } else { - Dbg(dbg_ctl, "Failed to get ciphers."); - } -} - -static void -add_extensions(JA4::TLSClientHelloSummary &summary, TSClientHello ch) -{ - for (auto ext_type : ch.get_extension_types()) { - summary.add_extension(ext_type); - } -} - -static std::string -hash_with_SHA256(std::string_view sv) -{ - Dbg(dbg_ctl, "Hashing %s", std::string{sv}.c_str()); - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256(reinterpret_cast<unsigned char const *>(sv.data()), sv.size(), hash); - std::string result; - result.resize(SHA256_DIGEST_LENGTH * 2 + 1); - for (int i{0}; i < SHA256_DIGEST_LENGTH; ++i) { - std::snprintf(result.data() + (i * 2), result.size() - (i * 2), "%02x", hash[i]); - } - return result; -} - -static std::string -get_fingerprint(TSClientHello ch) -{ - JA4::TLSClientHelloSummary summary{}; - summary.protocol = JA4::Protocol::TLS; - summary.TLS_version = get_version(ch); - summary.ALPN = get_first_ALPN(ch); - add_ciphers(summary, ch); - add_extensions(summary, ch); - std::string result{JA4::make_JA4_fingerprint(summary, hash_with_SHA256)}; - return result; -} - -void -ja4_method::on_client_hello(JAxContext *ctx, TSVConn vconn) -{ - TSClientHello ch = TSVConnClientHelloGet(vconn); - - if (!ch) { - Dbg(dbg_ctl, "Could not get TSClientHello object."); - } else { - ctx->set_fingerprint(get_fingerprint(ch)); - } -} diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h.h b/plugins/experimental/jax_fingerprint/ja4/method.cc similarity index 54% copy from plugins/experimental/jax_fingerprint/ja4h/ja4h.h copy to plugins/experimental/jax_fingerprint/ja4/method.cc index a49aa62c91..c41daba27e 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h.h +++ b/plugins/experimental/jax_fingerprint/ja4/method.cc @@ -17,25 +17,42 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ -#pragma once +#include "ts/ts.h" +#include <plugin.h> +#include <context.h> +#include "method.h" +#include "ja4.h" #include "datasource.h" +#include "tls_client_hello_summary.h" + +namespace ja4 +{ + +void +on_client_hello(JAxContext *ctx, TSVConn vconn) +{ + char fingerprint[ja4::FINGERPRINT_LENGTH]; + TSClientHello ch = TSVConnClientHelloGet(vconn); + + if (!ch) { + Dbg(dbg_ctl, "Could not get TSClientHello object."); + } else { + TLSClientHelloSummary datasource{ja4::Datasource::Protocol::TLS, ch}; + + generate_fingerprint(fingerprint, datasource); + + ctx->set_fingerprint({fingerprint, ja4::FINGERPRINT_LENGTH}); + } +} + +struct Method method = { + "JA4", + Method::Type::CONNECTION_BASED, + on_client_hello, + nullptr, +}; -constexpr int FINGERPRINT_LENGTH = 51; -constexpr size_t PART_A_POSITION = 0; -constexpr size_t PART_B_POSITION = 13; -constexpr size_t PART_C_POSITION = 26; -constexpr size_t PART_D_POSITION = 39; -constexpr size_t PART_A_LENGTH = 12; -constexpr size_t PART_B_LENGTH = 12; -constexpr size_t PART_C_LENGTH = 12; -constexpr size_t PART_D_LENGTH = 12; -constexpr char DELIMITER = '-'; -constexpr size_t DELIMITER_1_POSITION = 12; -constexpr size_t DELIMITER_2_POSITION = 25; -constexpr size_t DELIMITER_3_POSITION = 38; - -void generate_ja4h_fingerprint(char *out, Datasource &datasource); +} // namespace ja4 diff --git a/plugins/experimental/jax_fingerprint/ja3/ja3_method.h b/plugins/experimental/jax_fingerprint/ja4/method.h similarity index 97% rename from plugins/experimental/jax_fingerprint/ja3/ja3_method.h rename to plugins/experimental/jax_fingerprint/ja4/method.h index 0797113341..d85c5c2187 100644 --- a/plugins/experimental/jax_fingerprint/ja3/ja3_method.h +++ b/plugins/experimental/jax_fingerprint/ja4/method.h @@ -24,7 +24,7 @@ #include <method.h> -namespace ja3_method +namespace ja4 { extern struct Method method; diff --git a/plugins/experimental/jax_fingerprint/ja4/test.cc b/plugins/experimental/jax_fingerprint/ja4/test.cc new file mode 100644 index 0000000000..35f8e79b53 --- /dev/null +++ b/plugins/experimental/jax_fingerprint/ja4/test.cc @@ -0,0 +1,532 @@ +/** @file + * + Unit tests for JA4 fingerprint calculation. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include "ja4.h" +#include "datasource.h" + +#include <catch2/catch_test_macros.hpp> +#include <openssl/sha.h> + +#include <algorithm> +#include <cctype> +#include <cstring> +#include <string> +#include <string_view> +#include <unordered_map> +#include <vector> + +namespace +{ + +class MockDatasource : public ja4::Datasource +{ +public: + std::string_view + get_first_alpn() override + { + return this->_first_alpn; + } + + void + get_cipher_suites_hash(unsigned char out[32]) override + { + if (this->_ciphers.empty()) { + memset(out, 0, 32); + return; + } + auto sorted = this->_ciphers; + std::sort(sorted.begin(), sorted.end()); + SHA256_CTX ctx; + SHA256_Init(&ctx); + for (size_t i = 0; i < sorted.size(); ++i) { + char buf[5]; + char *p = buf; + if (i != 0) { + *p = ','; + p += 1; + } + uint16_t c = sorted[i]; + uint8_t h1 = (c & 0xF000) >> 12; + uint8_t l1 = (c & 0x0F00) >> 8; + uint8_t h2 = (c & 0x00F0) >> 4; + uint8_t l2 = c & 0x000F; + p[0] = h1 <= 9 ? ('0' + h1) : ('a' + h1 - 10); + p[1] = l1 <= 9 ? ('0' + l1) : ('a' + l1 - 10); + p[2] = h2 <= 9 ? ('0' + h2) : ('a' + h2 - 10); + p[3] = l2 <= 9 ? ('0' + l2) : ('a' + l2 - 10); + p += 4; + SHA256_Update(&ctx, buf, p - buf); + } + SHA256_Final(out, &ctx); + } + + void + get_extension_hash(unsigned char out[32]) override + { + if (this->_extensions.empty()) { + memset(out, 0, 32); + return; + } + auto sorted = this->_extensions; + std::sort(sorted.begin(), sorted.end()); + SHA256_CTX ctx; + SHA256_Init(&ctx); + for (size_t i = 0; i < sorted.size(); ++i) { + char buf[5]; + char *p = buf; + if (i != 0) { + *p = ','; + p += 1; + } + uint16_t e = sorted[i]; + uint8_t h1 = (e & 0xF000) >> 12; + uint8_t l1 = (e & 0x0F00) >> 8; + uint8_t h2 = (e & 0x00F0) >> 4; + uint8_t l2 = e & 0x000F; + p[0] = h1 <= 9 ? ('0' + h1) : ('a' + h1 - 10); + p[1] = l1 <= 9 ? ('0' + l1) : ('a' + l1 - 10); + p[2] = h2 <= 9 ? ('0' + h2) : ('a' + h2 - 10); + p[3] = l2 <= 9 ? ('0' + l2) : ('a' + l2 - 10); + p += 4; + SHA256_Update(&ctx, buf, p - buf); + } + SHA256_Final(out, &ctx); + } + + void + set_protocol(ja4::Datasource::Protocol protocol) + { + this->_protocol = protocol; + } + void + set_version(int version) + { + this->_version = version; + } + void + set_first_alpn(std::string first_alpn) + { + this->_first_alpn = first_alpn; + } + void + add_cipher(std::uint16_t cipher) + { + if (_is_GREASE(cipher)) { + return; + } + + ++this->_n_ciphers; + this->_ciphers.push_back(cipher); + } + + void + add_extension(uint16_t extension) + { + if (EXT_SNI == extension) { + this->_SNI_type = SNI::to_domain; + this->_has_SNI = true; + return; + } + if (EXT_ALPN == extension) { + this->_has_ALPN = true; + return; + } + if (_is_GREASE(extension)) { + return; + } + + ++this->_n_extensions; + this->_extensions.push_back(extension); + } + +private: + std::string _first_alpn; + + std::vector<std::uint16_t> _ciphers; + std::vector<std::uint16_t> _extensions; + SNI _SNI_type{SNI::to_IP}; +}; + +std::string_view +SHA256_12(std::string_view in) +{ + uint8_t hash[32]; + SHA256(reinterpret_cast<const uint8_t *>(in.data()), in.size(), hash); + + static char out[12]; + for (int i = 0; i < 6; ++i) { + uint8_t h = hash[i] >> 4; + uint8_t l = hash[i] & 0x0F; + out[i * 2] = h <= 9 ? '0' + h : 'a' + h - 10; + out[i * 2 + 1] = l <= 9 ? '0' + l : 'a' + l - 10; + } + return {out, sizeof(out)}; +} + +} // namespace + +static std::string call_JA4(ja4::Datasource &datasource); + +TEST_CASE("JA4") +{ + MockDatasource datasource{}; + + SECTION("Given the protocol is TLSP, " + "when we create a JA4 fingerprint, " + "then the first character thereof should be 't'.") + { + datasource.set_protocol(ja4::Datasource::Protocol::TLS); + + CHECK("t" == call_JA4(datasource).substr(0, 1)); + } + + SECTION("Given the protocol is QUIC, " + "when we create a JA4 fingerprint, " + "then the first character thereof should be 'q'.") + { + datasource.set_protocol(ja4::Datasource::Protocol::QUIC); + CHECK(call_JA4(datasource).starts_with('q')); + } + + SECTION("Given the protocol is DTLS, " + "when we create a JA4 fingerprint, " + "then the first character thereof should be 'd'.") + { + datasource.set_protocol(ja4::Datasource::Protocol::DTLS); + CHECK(call_JA4(datasource).starts_with('d')); + } + + SECTION("Given the TLS version is unknown, " + "when we create a JA4 fingerprint, " + "then indices [1,2] thereof should contain \"00\".") + { + datasource.set_version(0x123); + CHECK("00" == call_JA4(datasource).substr(1, 2)); + datasource.set_version(0x234); + CHECK("00" == call_JA4(datasource).substr(1, 2)); + } + + SECTION("Given the TLS version is known, " + "when we create a JA4 fingerprint, " + "then indices [1,2] thereof should contain the correct value.") + { + std::unordered_map<std::uint16_t, std::string> values{ + {0x304, "13"}, + {0x303, "12"}, + {0x302, "11"}, + {0x301, "10"}, + {0x300, "s3"}, + {0x200, "s2"}, + {0x100, "s1"}, + {0xfeff, "d1"}, + {0xfefd, "d2"}, + {0xfefc, "d3"} + }; + for (auto const &[version, expected] : values) { + CAPTURE(version, expected); + datasource.set_version(version); + CHECK(expected == call_JA4(datasource).substr(1, 2)); + } + } + + SECTION("Given the SNI extension is present, " + "when we create a JA4 fingerprint, " + "then index 3 thereof should contain 'd'.") + { + datasource.add_extension(0x0); + CHECK("d" == call_JA4(datasource).substr(3, 1)); + } + + SECTION("Given the SNI extension is not present, " + "when we create a JA4 fingerprint, " + "then index 3 thereof should contain 'i'.") + { + datasource.add_extension(0x31); + CHECK("i" == call_JA4(datasource).substr(3, 1)); + } + + SECTION("Given there is one cipher, " + "when we create a JA4 fingerprint, " + "then indices [4,5] thereof should contain \"01\".") + { + datasource.add_cipher(1); + CHECK("01" == call_JA4(datasource).substr(4, 2)); + } + + SECTION("Given there are 9 ciphers, " + "when we create a JA4 fingerprint, " + "then indices [4,5] thereof should contain \"09\".") + { + for (int i{0}; i < 9; ++i) { + datasource.add_cipher(i); + } + CHECK("09" == call_JA4(datasource).substr(4, 2)); + } + + SECTION("Given there are 10 ciphers, " + "when we create a JA4 fingerprint, " + "then indices [4,5] thereof should contain \"10\".") + { + for (int i{0}; i < 10; ++i) { + datasource.add_cipher(i); + } + CHECK("10" == call_JA4(datasource).substr(4, 2)); + } + + SECTION("Given there are more than 99 ciphers, " + "when we create a JA4 fingerprint, " + "then indices [4,5] thereof should contain \"99\".") + { + for (int i{0}; i < 100; ++i) { + datasource.add_cipher(i); + } + CHECK("99" == call_JA4(datasource).substr(4, 2)); + } + + SECTION("Given the ciphers include a GREASE value, " + "when we create a JA4 fingerprint, " + "then that value should not be included in the count.") + { + datasource.add_cipher(0x0a0a); + datasource.add_cipher(72); + CHECK("01" == call_JA4(datasource).substr(4, 2)); + } + + SECTION("Given there are no extensions, " + "when we create a JA4 fingerprint, " + "then indices [6,7] thereof should contain \"00\".") + { + CHECK("00" == call_JA4(datasource).substr(6, 2)); + } + + SECTION("Given there are 9 extensions, " + "when we create a JA4 fingerprint, " + "then indices [6,7] thereof should contain \"09\".") + { + for (int i{0}; i < 9; ++i) { + datasource.add_extension(i); + } + CHECK("09" == call_JA4(datasource).substr(6, 2)); + } + + SECTION("Given there are 99 extensions, " + "when we create a JA4 fingerprint, " + "then indices [6,7] thereof should contain \"99\".") + { + for (int i{0}; i < 99; ++i) { + datasource.add_extension(i); + } + CHECK("99" == call_JA4(datasource).substr(6, 2)); + } + + SECTION("Given there are more than 99 extensions, " + "when we create a JA4 fingerprint, " + "then indices [6,7] thereof should contain \"99\".") + { + for (int i{0}; i < 100; ++i) { + datasource.add_extension(i); + } + CHECK("99" == call_JA4(datasource).substr(6, 2)); + } + + SECTION("Given the extensions include a GREASE value, " + "when we create a JA4 fingerprint, " + "then that value should not be included in the count.") + { + datasource.add_extension(2); + datasource.add_extension(0x0a0a); + CHECK("01" == call_JA4(datasource).substr(6, 2)); + } + + // These may be covered by the earlier tests as well, but this documents the + // behavior explicitly. + SECTION("When we create a JA4 fingerprint, " + "then the SNI and ALPN extensions should be included in the count.") + { + datasource.add_extension(0x0); + datasource.add_extension(0x10); + CHECK("02" == call_JA4(datasource).substr(6, 2)); + } + + SECTION("Given the ALPN value is empty, " + "when we create a JA4 fingerprint, " + "then indices [8,9] thereof should contain \"00\".") + { + datasource.set_first_alpn(""); + CHECK("00" == call_JA4(datasource).substr(8, 2)); + } + + // This should never happen in practice because all registered ALPN values + // are at least 2 characters long, but it's the correct behavior according + // to the spec. :-) + SECTION("Given the ALPN value is \"a\", " + "when we create a JA4 fingerprint, " + "then indices [8,9] thereof should contain \"aa\".") + { + datasource.set_first_alpn("a"); + CHECK("aa" == call_JA4(datasource).substr(8, 2)); + } + + SECTION("Given the ALPN value is \"h3\", " + "when we create a JA4 fingerprint, " + "then indices [8,9] thereof should contain \"h3\".") + { + datasource.set_first_alpn("h3"); + CHECK("h3" == call_JA4(datasource).substr(8, 2)); + } + + SECTION("Given the ALPN value is \"imap\", " + "when we create a JA4 fingerprint, " + "then indices [8,9] thereof should contain \"ip\".") + { + datasource.set_first_alpn("imap"); + CHECK("ip" == call_JA4(datasource).substr(8, 2)); + } + + SECTION("When we create a JA4 fingerprint, " + "then index 10 thereof should contain '_'.") + { + CHECK("_" == call_JA4(datasource).substr(10, 1)); + } + + SECTION("When we create a JA4 fingerprint, " + "then the b section should be passed through the hash function.") + { + char buf[36]; + datasource.add_cipher(10); + CHECK(SHA256_12("000a") == ja4::generate_fingerprint(buf, datasource).substr(11, 12)); + } + + // As per the spec, we expect 4-character, comma-delimited hex values. + SECTION("Given only ciphers 2, 12, and 17 in that order, " + "when we create a JA4 fingerprint, " + "then the hash should be invoked with \"0002,000c,0011\".") + { + datasource.add_cipher(2); + datasource.add_cipher(12); + datasource.add_cipher(17); + char buf[36]; + CHECK(SHA256_12("0002,000c,0011") == ja4::generate_fingerprint(buf, datasource).substr(11, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then the cipher values should be sorted before hashing.") + { + datasource.add_cipher(17); + datasource.add_cipher(2); + datasource.add_cipher(12); + char buf[36]; + CHECK(SHA256_12("0002,000c,0011") == ja4::generate_fingerprint(buf, datasource).substr(11, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then GREASE values in the cipher list should be ignored.") + { + datasource.add_cipher(0x0a0a); + datasource.add_cipher(2); + char buf[36]; + CHECK(SHA256_12("0002") == ja4::generate_fingerprint(buf, datasource).substr(11, 12)); + } + + // All the tests from now on have enough ciphers to ensure a long enough + // hash using our default hash (the id function) so that the length of the + // JA4 fingerprint will be valid. + datasource.add_cipher(1); + datasource.add_cipher(2); + datasource.add_cipher(3); + + SECTION("When we create a JA4 fingerprint, " + "then we should truncate the section b hash to 12 characters.") + { + char buf[36]; + CHECK(SHA256_12("0001,0002,0003") == ja4::generate_fingerprint(buf, datasource).substr(11, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then index 10 thereof should contain '_'.") + { + CHECK("_" == call_JA4(datasource).substr(23, 1)); + } + + SECTION("When we create a JA4 fingerprint, " + "then the c section should be passed through the hash function.") + { + datasource.add_extension(10); + char buf[36]; + CHECK(SHA256_12("000a") == ja4::generate_fingerprint(buf, datasource).substr(24, 12)); + } + + // As per the spec, we expect 4-character, comma-delimited hex values. + SECTION("Given only extensions 2, 12, and 17 in that order, " + "when we create a JA4 fingerprint, " + "then the hash should be invoked with \"0002,000c,0011\".") + { + datasource.add_extension(2); + datasource.add_extension(12); + datasource.add_extension(17); + + char buf[36]; + CHECK(SHA256_12("0002,000c,0011") == ja4::generate_fingerprint(buf, datasource).substr(24, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then the extension values should be sorted before hashing.") + { + datasource.add_extension(17); + datasource.add_extension(2); + datasource.add_extension(12); + + char buf[36]; + CHECK(SHA256_12("0002,000c,0011") == ja4::generate_fingerprint(buf, datasource).substr(24, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then we ignore GREASE, SNI, ALPN, and SNI values in the extensions.") + { + datasource.add_extension(0x0a0a); + datasource.add_extension(0x0); + datasource.add_extension(0x10); + datasource.add_extension(5); + + char buf[36]; + CHECK(SHA256_12("0005") == ja4::generate_fingerprint(buf, datasource).substr(24, 12)); + } + + SECTION("When we create a JA4 fingerprint, " + "then we total length of the fingerprint should be 36 characters.") + { + datasource.add_extension(1); + datasource.add_extension(2); + datasource.add_extension(3); + CHECK(36 == call_JA4(datasource).size()); + } +} + +std::string +call_JA4(ja4::Datasource &datasource) +{ + char buf[36]; + ja4::generate_fingerprint(buf, datasource); + return {buf, 36}; +} diff --git a/plugins/experimental/jax_fingerprint/ja4/test_ja4.cc b/plugins/experimental/jax_fingerprint/ja4/test_ja4.cc deleted file mode 100644 index 8860d105f1..0000000000 --- a/plugins/experimental/jax_fingerprint/ja4/test_ja4.cc +++ /dev/null @@ -1,425 +0,0 @@ -/** @file - * - Unit tests for JA4 fingerprint calculation. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - */ - -#include "ja4.h" - -#include <catch2/catch_test_macros.hpp> - -#include <algorithm> -#include <cctype> -#include <string> -#include <string_view> -#include <unordered_map> - -static std::string call_JA4(JA4::TLSClientHelloSummary const &TLS_summary); -static std::string inc(std::string_view sv); - -TEST_CASE("JA4") -{ - JA4::TLSClientHelloSummary TLS_summary{}; - - SECTION("Given the protocol is TCP, " - "when we create a JA4 fingerprint, " - "then the first character thereof should be 't'.") - { - TLS_summary.protocol = JA4::Protocol::TLS; - - CHECK("t" == call_JA4(TLS_summary).substr(0, 1)); - } - - SECTION("Given the protocol is QUIC, " - "when we create a JA4 fingerprint, " - "then the first character thereof should be 'q'.") - { - TLS_summary.protocol = JA4::Protocol::QUIC; - CHECK(call_JA4(TLS_summary).starts_with('q')); - } - - SECTION("Given the protocol is DTLS, " - "when we create a JA4 fingerprint, " - "then the first character thereof should be 'd'.") - { - TLS_summary.protocol = JA4::Protocol::DTLS; - CHECK(call_JA4(TLS_summary).starts_with('d')); - } - - SECTION("Given the TLS version is unknown, " - "when we create a JA4 fingerprint, " - "then indices [1,2] thereof should contain \"00\".") - { - TLS_summary.TLS_version = 0x123; - CHECK("00" == call_JA4(TLS_summary).substr(1, 2)); - TLS_summary.TLS_version = 0x234; - CHECK("00" == call_JA4(TLS_summary).substr(1, 2)); - } - - SECTION("Given the TLS version is known, " - "when we create a JA4 fingerprint, " - "then indices [1,2] thereof should contain the correct value.") - { - std::unordered_map<std::uint16_t, std::string> values{ - {0x304, "13"}, - {0x303, "12"}, - {0x302, "11"}, - {0x301, "10"}, - {0x300, "s3"}, - {0x200, "s2"}, - {0x100, "s1"}, - {0xfeff, "d1"}, - {0xfefd, "d2"}, - {0xfefc, "d3"} - }; - for (auto const &[version, expected] : values) { - CAPTURE(version, expected); - TLS_summary.TLS_version = version; - CHECK(expected == call_JA4(TLS_summary).substr(1, 2)); - } - } - - SECTION("Given the SNI extension is present, " - "when we create a JA4 fingerprint, " - "then index 3 thereof should contain 'd'.") - { - TLS_summary.add_extension(0x0); - CHECK("d" == call_JA4(TLS_summary).substr(3, 1)); - } - - SECTION("Given the SNI extension is not present, " - "when we create a JA4 fingerprint, " - "then index 3 thereof should contain 'i'.") - { - TLS_summary.add_extension(0x31); - CHECK("i" == call_JA4(TLS_summary).substr(3, 1)); - } - - SECTION("Given there is one cipher, " - "when we create a JA4 fingerprint, " - "then indices [4,5] thereof should contain \"01\".") - { - TLS_summary.add_cipher(1); - CHECK("01" == call_JA4(TLS_summary).substr(4, 2)); - } - - SECTION("Given there are 9 ciphers, " - "when we create a JA4 fingerprint, " - "then indices [4,5] thereof should contain \"09\".") - { - for (int i{0}; i < 9; ++i) { - TLS_summary.add_cipher(i); - } - CHECK("09" == call_JA4(TLS_summary).substr(4, 2)); - } - - SECTION("Given there are 10 ciphers, " - "when we create a JA4 fingerprint, " - "then indices [4,5] thereof should contain \"10\".") - { - for (int i{0}; i < 10; ++i) { - TLS_summary.add_cipher(i); - } - CHECK("10" == call_JA4(TLS_summary).substr(4, 2)); - } - - SECTION("Given there are more than 99 ciphers, " - "when we create a JA4 fingerprint, " - "then indices [4,5] thereof should contain \"99\".") - { - for (int i{0}; i < 100; ++i) { - TLS_summary.add_cipher(i); - } - CHECK("99" == call_JA4(TLS_summary).substr(4, 2)); - } - - SECTION("Given the ciphers include a GREASE value, " - "when we create a JA4 fingerprint, " - "then that value should not be included in the count.") - { - TLS_summary.add_cipher(0x0a0a); - TLS_summary.add_cipher(72); - CHECK("01" == call_JA4(TLS_summary).substr(4, 2)); - } - - SECTION("Given there are no extensions, " - "when we create a JA4 fingerprint, " - "then indices [6,7] thereof should contain \"00\".") - { - CHECK("00" == call_JA4(TLS_summary).substr(6, 2)); - } - - SECTION("Given there are 9 extensions, " - "when we create a JA4 fingerprint, " - "then indices [6,7] thereof should contain \"09\".") - { - for (int i{0}; i < 9; ++i) { - TLS_summary.add_extension(i); - } - CHECK("09" == call_JA4(TLS_summary).substr(6, 2)); - } - - SECTION("Given there are 99 extensions, " - "when we create a JA4 fingerprint, " - "then indices [6,7] thereof should contain \"99\".") - { - for (int i{0}; i < 99; ++i) { - TLS_summary.add_extension(i); - } - CHECK("99" == call_JA4(TLS_summary).substr(6, 2)); - } - - SECTION("Given there are more than 99 extensions, " - "when we create a JA4 fingerprint, " - "then indices [6,7] thereof should contain \"99\".") - { - for (int i{0}; i < 100; ++i) { - TLS_summary.add_extension(i); - } - CHECK("99" == call_JA4(TLS_summary).substr(6, 2)); - } - - SECTION("Given the extensions include a GREASE value, " - "when we create a JA4 fingerprint, " - "then that value should not be included in the count.") - { - TLS_summary.add_extension(2); - TLS_summary.add_extension(0x0a0a); - CHECK("01" == call_JA4(TLS_summary).substr(6, 2)); - } - - // These may be covered by the earlier tests as well, but this documents the - // behavior explicitly. - SECTION("When we create a JA4 fingerprint, " - "then the SNI and ALPN extensions should be included in the count.") - { - TLS_summary.add_extension(0x0); - TLS_summary.add_extension(0x10); - CHECK("02" == call_JA4(TLS_summary).substr(6, 2)); - } - - SECTION("Given the ALPN value is empty, " - "when we create a JA4 fingerprint, " - "then indices [8,9] thereof should contain \"00\".") - { - TLS_summary.ALPN = ""; - CHECK("00" == call_JA4(TLS_summary).substr(8, 2)); - } - - // This should never happen in practice because all registered ALPN values - // are at least 2 characters long, but it's the correct behavior according - // to the spec. :-) - SECTION("Given the ALPN value is \"a\", " - "when we create a JA4 fingerprint, " - "then indices [8,9] thereof should contain \"aa\".") - { - TLS_summary.ALPN = 'a'; - CHECK("aa" == call_JA4(TLS_summary).substr(8, 2)); - } - - SECTION("Given the ALPN value is \"h3\", " - "when we create a JA4 fingerprint, " - "then indices [8,9] thereof should contain \"h3\".") - { - TLS_summary.ALPN = "h3"; - CHECK("h3" == call_JA4(TLS_summary).substr(8, 2)); - } - - SECTION("Given the ALPN value is \"imap\", " - "when we create a JA4 fingerprint, " - "then indices [8,9] thereof should contain \"ip\".") - { - TLS_summary.ALPN = "imap"; - CHECK("ip" == call_JA4(TLS_summary).substr(8, 2)); - } - - SECTION("When we create a JA4 fingeprint, " - "then index 10 thereof should contain '_'.") - { - CHECK("_" == call_JA4(TLS_summary).substr(10, 1)); - } - - SECTION("When we create a JA4 fingerprint, " - "then the b section should be passed through the hash function.") - { - TLS_summary.add_cipher(10); - CHECK("111b" == JA4::make_JA4_fingerprint(TLS_summary, [](std::string_view sv) { return inc(sv); }).substr(11, 4)); - } - - // As per the spec, we expect 4-character, comma-delimited hex values. - SECTION("Given only ciphers 2, 12, and 17 in that order, " - "when we create a JA4 fingerprint, " - "then the hash should be invoked with \"0002,000c,0011\".") - { - TLS_summary.add_cipher(2); - TLS_summary.add_cipher(12); - TLS_summary.add_cipher(17); - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0002,000c,0011" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - SECTION("When we create a JA4 fingerprint, " - "then the cipher values should be sorted before hashing.") - { - TLS_summary.add_cipher(17); - TLS_summary.add_cipher(2); - TLS_summary.add_cipher(12); - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0002,000c,0011" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - SECTION("When we create a JA4 fingerprint, " - "then GREASE values in the cipher list should be ignored.") - { - TLS_summary.add_cipher(0x0a0a); - TLS_summary.add_cipher(2); - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0002" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - // All the tests from now on have enough ciphers to ensure a long enough - // hash using our default hash (the id function) so that the length of the - // JA4 fingerprint will be valid. - TLS_summary.add_cipher(1); - TLS_summary.add_cipher(2); - TLS_summary.add_cipher(3); - - SECTION("When we create a JA4 fingerprint, " - "then we should truncate the section b hash to 12 characters.") - { - CHECK("001,0002,000_" == JA4::make_JA4_fingerprint(TLS_summary, [](std::string_view sv) { - return sv.empty() ? sv : sv.substr(1); - }).substr(11, 13)); - } - - SECTION("When we create a JA4 fingeprint, " - "then index 10 thereof should contain '_'.") - { - CHECK("_" == call_JA4(TLS_summary).substr(23, 1)); - } - - SECTION("When we create a JA4 fingerprint, " - "then the c section should be passed through the hash function.") - { - TLS_summary.add_extension(10); - CHECK("111b" == JA4::make_JA4_fingerprint(TLS_summary, [](std::string_view sv) { return inc(sv); }).substr(24, 4)); - } - - // As per the spec, we expect 4-character, comma-delimited hex values. - SECTION("Given only extensions 2, 12, and 17 in that order, " - "when we create a JA4 fingerprint, " - "then the hash should be invoked with \"0002,000c,0011\".") - { - TLS_summary.add_extension(2); - TLS_summary.add_extension(12); - TLS_summary.add_extension(17); - - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0002,000c,0011" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - SECTION("When we create a JA4 fingerprint, " - "then the extension values should be sorted before hashing.") - { - TLS_summary.add_extension(17); - TLS_summary.add_extension(2); - TLS_summary.add_extension(12); - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0002,000c,0011" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - SECTION("When we create a JA4 fingerprint, " - "then we ignore GREASE, SNI, ALPN, and SNI values in the extensions.") - { - TLS_summary.add_extension(0x0a0a); - TLS_summary.add_extension(0x0); - TLS_summary.add_extension(0x10); - TLS_summary.add_extension(5); - bool verified{false}; - // INFO doesn't work from inside the lambda body. :/ - JA4::make_JA4_fingerprint(TLS_summary, [&verified](std::string_view sv) { - if ("0005" == sv) { - verified = true; - } - return sv; - }); - CHECK(verified); - } - - SECTION("When we create a JA4 fingerprint, " - "then we total length of the fingerprint should be 36 characters.") - { - TLS_summary.add_extension(1); - TLS_summary.add_extension(2); - TLS_summary.add_extension(3); - CHECK(36 == call_JA4(TLS_summary).size()); - } -} - -std::string -call_JA4(JA4::TLSClientHelloSummary const &TLS_summary) -{ - return JA4::make_JA4_fingerprint(TLS_summary, [](std::string_view sv) { return sv; }); -} - -std::string -inc(std::string_view sv) -{ - std::string result; - result.resize(sv.size()); - std::transform(sv.begin(), sv.end(), result.begin(), [](char c) { return c + 1; }); - return result; -} diff --git a/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.cc b/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.cc index cc35a1c83b..b3607ac379 100644 --- a/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.cc +++ b/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.cc @@ -22,91 +22,184 @@ */ +#include "ts/ts.h" +#include <plugin.h> #include "ja4.h" -#include <algorithm> -#include <array> +#include "tls_client_hello_summary.h" + +#include <openssl/sha.h> #include <cstdint> -#include <functional> -#include <vector> +#include <algorithm> -namespace +TLSClientHelloSummary::TLSClientHelloSummary(ja4::Datasource::Protocol protocol, TSClientHello ch) : _ch(ch) { + const uint8_t *buf; + size_t buflen; + + // Protocol + this->_protocol = protocol; + + // Version + if (TS_SUCCESS == TSClientHelloExtensionGet(this->_ch, EXT_SUPPORTED_VERSIONS, &buf, &buflen)) { + if (buflen == 0 || buflen < static_cast<unsigned int>(buf[0] + 1)) { + Dbg(dbg_ctl, "Malformed supported_versions extension (truncated vector)... using legacy version."); + this->_version = this->_ch.get_version(); + } else { + uint16_t max_version = 0; + size_t versions_len = buf[0]; + for (size_t i = 1; (i + 1) < (versions_len + 1); i += 2) { + uint16_t version = (buf[i] << 8) | buf[i + 1]; + if (!this->_is_GREASE(version) && version > max_version) { + max_version = version; + } + } + this->_version = max_version; + } + } else { + Dbg(dbg_ctl, "No supported_versions extension... using legacy version."); + this->_version = this->_ch.get_version(); + } -constexpr std::array<std::uint16_t, 16> GREASE_values{0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, - 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba, 0xcaca, 0xdada, 0xeaea, 0xfafa}; -constexpr std::uint16_t extension_SNI{0x0}; -constexpr std::uint16_t extension_ALPN{0x10}; - -} // end anonymous namespace + // Ciphers + buf = this->_ch.get_cipher_suites(); + buflen = this->_ch.get_cipher_suites_len(); + + if (buflen / 2 <= MAX_CIPHERS_FOR_FAST_PATH) { + // Fast path + this->_ciphers = this->_fast_cipher_storage.data(); + } else { + // Slow path + this->_slow_cipher_storage = std::make_unique<uint16_t[]>(buflen / 2); + this->_ciphers = this->_slow_cipher_storage.get(); + } + for (size_t i = 0; i + 1 < buflen; i += 2) { + uint16_t cipher = (static_cast<uint16_t>(buf[i]) << 8) + buf[i + 1]; + if (this->_is_GREASE(cipher)) { + continue; + } + this->_ciphers[this->_n_ciphers++] = cipher; + } + std::sort(this->_ciphers, this->_ciphers + this->_n_ciphers); -static bool is_ignored_non_GREASE_extension(std::uint16_t extension); + // Extensions + auto count = 0; + for (auto &&type : this->_ch.get_extension_types()) { + (void)type; + ++count; + } + if (count <= MAX_EXTENSIONS_FOR_FAST_PATH) { + // Fast path + this->_extensions = this->_fast_extension_storage.data(); + } else { + // Slow path + this->_slow_extension_storage = std::make_unique<uint16_t[]>(count); + this->_extensions = this->_slow_extension_storage.get(); + } + for (auto &&type : this->_ch.get_extension_types()) { + if (type == EXT_SNI) { + this->_has_SNI = true; + continue; + } + if (type == EXT_ALPN) { + this->_has_ALPN = true; + continue; + } + if (this->_is_GREASE(type)) { + continue; + } + this->_extensions[this->_n_extensions++] = type; + } + std::sort(this->_extensions, this->_extensions + this->_n_extensions); +} -std::vector<std::uint16_t> const & -JA4::TLSClientHelloSummary::get_ciphers() const +std::string_view +TLSClientHelloSummary::get_first_alpn() { - return this->_ciphers; + unsigned char const *buf{}; + std::size_t buflen{}; + + if (TS_SUCCESS == TSClientHelloExtensionGet(this->_ch, EXT_ALPN, &buf, &buflen)) { + // The first two bytes are a 16bit encoding of the total length. + if (buflen < 4) { + return {}; + } + + unsigned char first_ALPN_length = buf[2]; + if (first_ALPN_length == 0 || first_ALPN_length > (buflen - 3)) { + return {}; + } + + return {reinterpret_cast<const char *>(&(buf[3])), first_ALPN_length}; + } else { + return {}; + } } void -JA4::TLSClientHelloSummary::add_cipher(std::uint16_t cipher) +TLSClientHelloSummary::get_cipher_suites_hash(unsigned char out[32]) { - if (is_GREASE(cipher)) { + if (this->_n_ciphers == 0) { + memset(out, 0, 32); return; } - this->_ciphers.push_back(cipher); -} + SHA256_CTX sha256ctx; + SHA256_Init(&sha256ctx); + + for (int i = 0; i < this->_n_ciphers; ++i) { + char buf[5]; + char *p = buf; + if (i != 0) { + *p = ','; + p += 1; + } + uint16_t &cipher = this->_ciphers[i]; + uint8_t h1 = (cipher & 0xF000) >> 12; + uint8_t l1 = (cipher & 0x0F00) >> 8; + uint8_t h2 = (cipher & 0x00F0) >> 4; + uint8_t l2 = cipher & 0x000F; + p[0] = h1 <= 9 ? ('0' + h1) : ('a' + h1 - 10); + p[1] = l1 <= 9 ? ('0' + l1) : ('a' + l1 - 10); + p[2] = h2 <= 9 ? ('0' + h2) : ('a' + h2 - 10); + p[3] = l2 <= 9 ? ('0' + l2) : ('a' + l2 - 10); + p += 4; + SHA256_Update(&sha256ctx, buf, p - buf); + } -std::vector<std::uint16_t> const & -JA4::TLSClientHelloSummary::get_extensions() const -{ - return this->_extensions; + SHA256_Final(out, &sha256ctx); } void -JA4::TLSClientHelloSummary::add_extension(std::uint16_t extension) +TLSClientHelloSummary::get_extension_hash(unsigned char out[32]) { - if (is_GREASE(extension)) { + if (this->_n_extensions == 0) { + memset(out, 0, 32); return; } - if (extension_SNI == extension) { - this->_SNI_type = SNI::to_domain; + SHA256_CTX sha256ctx; + SHA256_Init(&sha256ctx); + + for (int i = 0; i < this->_n_extensions; ++i) { + char buf[5]; + char *p = buf; + if (i != 0) { + *p = ','; + p += 1; + } + uint16_t &extension = this->_extensions[i]; + uint8_t h1 = (extension & 0xF000) >> 12; + uint8_t l1 = (extension & 0x0F00) >> 8; + uint8_t h2 = (extension & 0x00F0) >> 4; + uint8_t l2 = extension & 0x000F; + p[0] = h1 <= 9 ? ('0' + h1) : ('a' + h1 - 10); + p[1] = l1 <= 9 ? ('0' + l1) : ('a' + l1 - 10); + p[2] = h2 <= 9 ? ('0' + h2) : ('a' + h2 - 10); + p[3] = l2 <= 9 ? ('0' + l2) : ('a' + l2 - 10); + p += 4; + SHA256_Update(&sha256ctx, buf, p - buf); } - ++this->_extension_count_including_sni_and_alpn; - if (!is_ignored_non_GREASE_extension(extension)) { - this->_extensions.push_back(extension); - } -} - -JA4::TLSClientHelloSummary::difference_type -JA4::TLSClientHelloSummary::get_cipher_count() const -{ - return this->_ciphers.size(); -} - -JA4::TLSClientHelloSummary::difference_type -JA4::TLSClientHelloSummary::get_extension_count() const -{ - return this->_extension_count_including_sni_and_alpn; -} - -bool -is_ignored_non_GREASE_extension(std::uint16_t extension) -{ - return (extension_SNI == extension) || (extension_ALPN == extension); -} - -JA4::SNI -JA4::TLSClientHelloSummary::get_SNI_type() const -{ - return this->_SNI_type; -} - -bool -JA4::is_GREASE(std::uint16_t value) -{ - return std::binary_search(GREASE_values.begin(), GREASE_values.end(), value); + SHA256_Final(out, &sha256ctx); } diff --git a/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.h b/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.h new file mode 100644 index 0000000000..165f274f00 --- /dev/null +++ b/plugins/experimental/jax_fingerprint/ja4/tls_client_hello_summary.h @@ -0,0 +1,55 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#pragma once + +#include <array> +#include <memory> +#include "ts/ts.h" +#include "datasource.h" + +/** + * Represents the data sent in a TLS Client Hello needed for JA4 fingerprints. + */ +class TLSClientHelloSummary : public ja4::Datasource +{ +public: + TLSClientHelloSummary(ja4::Datasource::Protocol protocol, TSClientHello ch); + + std::string_view get_first_alpn() override; + void get_cipher_suites_hash(unsigned char out[32]) override; + void get_extension_hash(unsigned char out[32]) override; + +private: + static constexpr int MAX_CIPHERS_FOR_FAST_PATH = 64; + static constexpr int MAX_EXTENSIONS_FOR_FAST_PATH = 32; + + TSClientHello _ch; + + uint16_t *_ciphers = nullptr; + std::array<uint16_t, MAX_CIPHERS_FOR_FAST_PATH> _fast_cipher_storage; + std::unique_ptr<uint16_t[]> _slow_cipher_storage; + + uint16_t *_extensions = nullptr; + std::array<uint16_t, MAX_EXTENSIONS_FOR_FAST_PATH> _fast_extension_storage; + std::unique_ptr<uint16_t[]> _slow_extension_storage; +}; diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h b/plugins/experimental/jax_fingerprint/ja4_common/utils.cc similarity index 68% copy from plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h copy to plugins/experimental/jax_fingerprint/ja4_common/utils.cc index e64901b815..c33a2c8efd 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h +++ b/plugins/experimental/jax_fingerprint/ja4_common/utils.cc @@ -1,4 +1,5 @@ /** @file + * @section license License @@ -20,13 +21,17 @@ */ -#pragma once +#include "utils.h" -#include <method.h> +constexpr int TRUNCATED_HASH_STRING_LENGTH = 6; -namespace ja4h_method +void +hash_stringify(char *out, const unsigned char *hash) { - -extern struct Method method; - + for (int i = 0; i < TRUNCATED_HASH_STRING_LENGTH; ++i) { + unsigned int h = hash[i] >> 4; + unsigned int l = hash[i] & 0x0F; + out[i * 2] = h <= 9 ? ('0' + h) : ('a' + h - 10); + out[i * 2 + 1] = l <= 9 ? ('0' + l) : ('a' + l - 10); + } } diff --git a/plugins/experimental/jax_fingerprint/ja4/ja4_method.h b/plugins/experimental/jax_fingerprint/ja4_common/utils.h similarity index 91% rename from plugins/experimental/jax_fingerprint/ja4/ja4_method.h rename to plugins/experimental/jax_fingerprint/ja4_common/utils.h index 4b7abea288..1dcfbaf5c7 100644 --- a/plugins/experimental/jax_fingerprint/ja4/ja4_method.h +++ b/plugins/experimental/jax_fingerprint/ja4_common/utils.h @@ -22,11 +22,4 @@ #pragma once -#include <method.h> - -namespace ja4_method -{ - -extern struct Method method; - -} +void hash_stringify(char *out, const unsigned char *hash); diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h.cc b/plugins/experimental/jax_fingerprint/ja4h/ja4h.cc index 7175797035..18d8f23fea 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h.cc +++ b/plugins/experimental/jax_fingerprint/ja4h/ja4h.cc @@ -106,7 +106,7 @@ generate_ja4h_d(char *out, Datasource & /* datasource ATS_UNUSED */) } void -generate_ja4h_fingerprint(char *out, Datasource &datasource) +ja4h::generate_fingerprint(char *out, Datasource &datasource) { generate_ja4h_a(&(out[PART_A_POSITION]), datasource); out[DELIMITER_1_POSITION] = DELIMITER; diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h.h b/plugins/experimental/jax_fingerprint/ja4h/ja4h.h index a49aa62c91..161840e878 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h.h +++ b/plugins/experimental/jax_fingerprint/ja4h/ja4h.h @@ -24,6 +24,9 @@ #include "datasource.h" +namespace ja4h +{ + constexpr int FINGERPRINT_LENGTH = 51; constexpr size_t PART_A_POSITION = 0; constexpr size_t PART_B_POSITION = 13; @@ -38,4 +41,6 @@ constexpr size_t DELIMITER_1_POSITION = 12; constexpr size_t DELIMITER_2_POSITION = 25; constexpr size_t DELIMITER_3_POSITION = 38; -void generate_ja4h_fingerprint(char *out, Datasource &datasource); +void generate_fingerprint(char *out, Datasource &datasource); + +} // end namespace ja4h diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.cc b/plugins/experimental/jax_fingerprint/ja4h/method.cc similarity index 95% rename from plugins/experimental/jax_fingerprint/ja4h/ja4h_method.cc rename to plugins/experimental/jax_fingerprint/ja4h/method.cc index c559b7cfbd..5bf182a79d 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.cc +++ b/plugins/experimental/jax_fingerprint/ja4h/method.cc @@ -23,22 +23,15 @@ #include <plugin.h> #include <context.h> -#include "ja4h_method.h" +#include <ja4_common/utils.h> +#include "method.h" #include "ja4h.h" #include "datasource.h" #include "openssl/sha.h" -namespace ja4h_method +namespace { -void on_request(JAxContext *, TSHttpTxn); - -struct Method method = { - "JA4H", - Method::Type::REQUEST_BASED, - nullptr, - on_request, -}; class TxnDatasource : public Datasource { @@ -159,15 +152,26 @@ TxnDatasource::get_headers_hash(unsigned char out[32]) SHA256_Final(out, &sha256ctx); } -} // namespace ja4h_method +} // namespace +namespace ja4h +{ void -ja4h_method::on_request(JAxContext *ctx, TSHttpTxn txnp) +on_request(JAxContext *ctx, TSHttpTxn txnp) { char fingerprint[FINGERPRINT_LENGTH]; TxnDatasource datasource{txnp}; - generate_ja4h_fingerprint(fingerprint, datasource); + generate_fingerprint(fingerprint, datasource); ctx->set_fingerprint({fingerprint, FINGERPRINT_LENGTH}); } + +struct Method method = { + "JA4H", + Method::Type::REQUEST_BASED, + nullptr, + on_request, +}; + +} // namespace ja4h diff --git a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h b/plugins/experimental/jax_fingerprint/ja4h/method.h similarity index 97% rename from plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h rename to plugins/experimental/jax_fingerprint/ja4h/method.h index e64901b815..fb6551ba32 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/ja4h_method.h +++ b/plugins/experimental/jax_fingerprint/ja4h/method.h @@ -24,7 +24,7 @@ #include <method.h> -namespace ja4h_method +namespace ja4h { extern struct Method method; diff --git a/plugins/experimental/jax_fingerprint/ja4h/test_ja4h.cc b/plugins/experimental/jax_fingerprint/ja4h/test.cc similarity index 68% rename from plugins/experimental/jax_fingerprint/ja4h/test_ja4h.cc rename to plugins/experimental/jax_fingerprint/ja4h/test.cc index 2ae5ef8878..d2f84b1a8b 100644 --- a/plugins/experimental/jax_fingerprint/ja4h/test_ja4h.cc +++ b/plugins/experimental/jax_fingerprint/ja4h/test.cc @@ -25,8 +25,12 @@ #include <openssl/sha.h> #include <catch2/catch_test_macros.hpp> +#include <string> #include <map> +namespace +{ + class MockDatasource : public Datasource { public: @@ -73,7 +77,7 @@ public: SHA256_CTX sha256ctx; SHA256_Init(&sha256ctx); - for (auto &&ite : this->_fields) { + for (auto ite : this->_fields) { if (this->_should_include_field({ite.first.c_str(), ite.first.size()})) { SHA256_Update(&sha256ctx, ite.first.c_str(), ite.first.size()); } @@ -101,7 +105,7 @@ public: private: std::string _method; int _version; - std::map<std::string, std::string> _fields; + std::map<std::string, std::string> _fields{}; }; std::string_view @@ -120,33 +124,36 @@ SHA256_12(std::string_view in) return {out, sizeof(out)}; } +} // namespace + TEST_CASE("JA4H") { - MockDatasource datasource; - char fingerprint[FINGERPRINT_LENGTH]; + char fingerprint[ja4h::FINGERPRINT_LENGTH]; SECTION("HTTP/1.0 GET with Host") { + MockDatasource datasource; datasource.set_method("GET"); datasource.set_version(1 << 16 | 0); datasource.set_fields({ {"Host", "abc.example"}, }); - generate_ja4h_fingerprint(fingerprint, datasource); + ja4h::generate_fingerprint(fingerprint, datasource); std::string_view fingerprint_sv{fingerprint, sizeof(fingerprint)}; - CHECK(fingerprint_sv.substr(PART_A_POSITION, PART_A_LENGTH) == "ge10nn010000"); - CHECK(fingerprint_sv[DELIMITER_1_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_B_POSITION, PART_B_LENGTH) == SHA256_12("Host")); - CHECK(fingerprint_sv[DELIMITER_2_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_C_POSITION, PART_C_LENGTH) == "000000000000"); - CHECK(fingerprint_sv[DELIMITER_3_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_D_POSITION, PART_D_LENGTH) == "000000000000"); + CHECK(fingerprint_sv.substr(ja4h::PART_A_POSITION, ja4h::PART_A_LENGTH) == "ge10nn010000"); + CHECK(fingerprint_sv[ja4h::DELIMITER_1_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_B_POSITION, ja4h::PART_B_LENGTH) == SHA256_12("Host")); + CHECK(fingerprint_sv[ja4h::DELIMITER_2_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_C_POSITION, ja4h::PART_C_LENGTH) == "000000000000"); + CHECK(fingerprint_sv[ja4h::DELIMITER_3_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_D_POSITION, ja4h::PART_D_LENGTH) == "000000000000"); } SECTION("HTTP/1.1 POST with Accept-Language and Referer") { + MockDatasource datasource; datasource.set_method("POST"); datasource.set_version(1 << 16 | 1); datasource.set_fields({ @@ -154,15 +161,15 @@ TEST_CASE("JA4H") {"Referer", "https://xyz.example/foo"}, }); - generate_ja4h_fingerprint(fingerprint, datasource); + ja4h::generate_fingerprint(fingerprint, datasource); std::string_view fingerprint_sv{fingerprint, sizeof(fingerprint)}; - CHECK(fingerprint_sv.substr(PART_A_POSITION, PART_A_LENGTH) == "po11nr02en00"); - CHECK(fingerprint_sv[DELIMITER_1_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_B_POSITION, PART_B_LENGTH) == SHA256_12("Accept-Language")); - CHECK(fingerprint_sv[DELIMITER_2_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_C_POSITION, PART_C_LENGTH) == "000000000000"); - CHECK(fingerprint_sv[DELIMITER_3_POSITION] == DELIMITER); - CHECK(fingerprint_sv.substr(PART_D_POSITION, PART_D_LENGTH) == "000000000000"); + CHECK(fingerprint_sv.substr(ja4h::PART_A_POSITION, ja4h::PART_A_LENGTH) == "po11nr02en00"); + CHECK(fingerprint_sv[ja4h::DELIMITER_1_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_B_POSITION, ja4h::PART_B_LENGTH) == SHA256_12("Accept-Language")); + CHECK(fingerprint_sv[ja4h::DELIMITER_2_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_C_POSITION, ja4h::PART_C_LENGTH) == "000000000000"); + CHECK(fingerprint_sv[ja4h::DELIMITER_3_POSITION] == ja4h::DELIMITER); + CHECK(fingerprint_sv.substr(ja4h::PART_D_POSITION, ja4h::PART_D_LENGTH) == "000000000000"); } } diff --git a/plugins/experimental/jax_fingerprint/plugin.cc b/plugins/experimental/jax_fingerprint/plugin.cc index 50f6353d8d..5044e43e52 100644 --- a/plugins/experimental/jax_fingerprint/plugin.cc +++ b/plugins/experimental/jax_fingerprint/plugin.cc @@ -27,9 +27,9 @@ #include "header.h" #include "log.h" -#include "ja4/ja4_method.h" -#include "ja4h/ja4h_method.h" -#include "ja3/ja3_method.h" +#include "ja4/method.h" +#include "ja4h/method.h" +#include "ja3/method.h" #include <ts/apidefs.h> #include <ts/ts.h> @@ -49,11 +49,6 @@ DbgCtl dbg_ctl{PLUGIN_NAME}; -namespace -{ - -} // end anonymous namespace - static bool read_config_option(int argc, char const *argv[], PluginConfig &config) { @@ -77,11 +72,11 @@ read_config_option(int argc, char const *argv[], PluginConfig &config) break; case 'M': if (strcmp("JA4", optarg) == 0) { - config.method = ja4_method::method; + config.method = ja4::method; } else if (strcmp("JA4H", optarg) == 0) { - config.method = ja4h_method::method; + config.method = ja4h::method; } else if (strcmp("JA3", optarg) == 0) { - config.method = ja3_method::method; + config.method = ja3::method; } else { Dbg(dbg_ctl, "Unexpected method: %s", optarg); return false;
