diff --git a/CMakeLists.txt b/CMakeLists.txt index 8272382e3..79c0fb2ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,8 +72,13 @@ if(${NO_EVENTFD}) endif() # Check packages +find_package(CURL 7.29 REQUIRED) +find_package(fmt REQUIRED) +find_package(OpenSSL 1.0.0 REQUIRED) find_package(PkgConfig REQUIRED) +find_package(spdlog REQUIRED) find_package(Threads REQUIRED) +find_package(nlohmann_json REQUIRED) find_package(OpenMP) find_package(IBVerbs) find_package(RDMACM) @@ -96,7 +101,6 @@ if(NOT PASTE) message(SEND_ERROR "GNU paste is missing. Please install coreutils") endif() - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/share/pkgconfig:/usr/lib64/pkgconfig") pkg_check_modules(JANSSON IMPORTED_TARGET REQUIRED jansson>=2.13) @@ -171,9 +175,10 @@ else() endif() # Build options -cmake_dependent_option(WITHOUT_GPL "Build VILLASnode without any GPL code" OFF "" ON) +option(LOG_COLOR_DISABLE "Disable any colored log output" OFF) +option(WITHOUT_GPL "Build VILLASnode without any GPL code" OFF) +option(WITH_DEFAULTS "Defaults for non required build options" ON) cmake_dependent_option(WITH_GHC_FS "Build using ghc::filesystem, a drop in replacement for std::filesystem" ON "STDCXX_FS_NOT_FOUND" OFF) -cmake_dependent_option(WITH_DEFAULTS "Defaults for non required build options" ON "" OFF) cmake_dependent_option(WITH_API "Build with remote control API" "${WITH_DEFAULTS}" "" OFF) cmake_dependent_option(WITH_CLIENTS "Build client applications" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) cmake_dependent_option(WITH_CONFIG "Build with support for libconfig configuration syntax" "${WITH_DEFAULTS}" "LIBCONFIG_FOUND" OFF) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f794a6950..5f1b17d73 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -4,33 +4,11 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 -include(FindPkgConfig) -include(FeatureSummary) -include(GNUInstallDirs) - -# Check packages -find_package(OpenSSL 1.0.0 REQUIRED) -find_package(CURL 7.29 REQUIRED) -find_package(spdlog 1.6.0 REQUIRED) -find_package(fmt 6.0.0 REQUIRED) - -pkg_check_modules(JANSSON IMPORTED_TARGET REQUIRED jansson>=2.7) -pkg_check_modules(LIBCONFIG IMPORTED_TARGET libconfig>=1.4.9) -pkg_check_modules(UUID IMPORTED_TARGET REQUIRED uuid>=2.23) - -if(fmt_VERSION VERSION_LESS "9.0.0") - message("Using legacy ostream formatting") - set(FMT_LEGACY_OSTREAM_FORMATTER 1) -endif() - add_subdirectory(lib) if(WITH_TESTS) add_subdirectory(tests) endif() -# Disable any colored log output -option(LOG_COLOR_DISABLE "Disable any colored log output" OFF) - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/villas/config.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/villas/config.hpp diff --git a/common/include/villas/jansson.hpp b/common/include/villas/jansson.hpp new file mode 100644 index 000000000..7d8db346e --- /dev/null +++ b/common/include/villas/jansson.hpp @@ -0,0 +1,516 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +namespace villas { + +// smart pointer for ::json_t values. +// +// this class mirrors the interface of std::shared_ptr using the internal reference count of ::json_t +class JanssonPtr { +public: + JanssonPtr() : inner(nullptr) {} + explicit JanssonPtr(::json_t *json) : inner(json) {} + + JanssonPtr(JanssonPtr const &other) : inner(json_incref(other.inner)) {} + JanssonPtr(JanssonPtr &&other) : inner(std::exchange(other.inner, nullptr)) {} + ~JanssonPtr() { json_decref(inner); } + + JanssonPtr &operator=(JanssonPtr const &other) { + json_decref(inner); + inner = json_incref(other.inner); + return *this; + } + + JanssonPtr &operator=(JanssonPtr &&other) { + json_decref(inner); + inner = std::exchange(other.inner, nullptr); + return *this; + } + + ::json_t *release() { return std::exchange(inner, nullptr); } + + void reset() { + json_decref(inner); + inner = nullptr; + } + + void reset(::json_t *json) { + json_decref(inner); + inner = json_incref(json); + } + + void swap(JanssonPtr &other) { std::swap(inner, other.inner); } + + operator bool() { return inner != nullptr; } + ::json_t *get() const { return inner; } + ::json_t *operator->() const { return inner; } + + friend void swap(JanssonPtr &lhs, JanssonPtr &rhs) { lhs.swap(rhs); } + friend auto operator<=>(JanssonPtr const &, JanssonPtr const &) = default; + +private: + ::json_t *inner; +}; + +// type trait helper for applying variadic default promotions +template class va_default_promote { + static auto impl() { + using U = std::decay_t; + if constexpr (std::is_enum_v) { + if constexpr (std::is_convertible_v>) { + // unscoped enumerations are converted to their underlying type and then promoted using integer promotions + return std::type_identity< + decltype(+std::declval>())>(); + } else { + // scoped enumeration handling is implementation defined just pass them without promotions + return std::type_identity(); + } + } else if constexpr (std::is_same_v) { + // float values are promoted to double + return std::type_identity(); + } else if constexpr (std::is_integral_v) { + // integral values are promoted using integer promotions + return std::type_identity())>(); + } else { + // default case without any promotions + return std::type_identity(); + } + } + +public: + using type = decltype(impl())::type; +}; + +template using va_default_promote_t = va_default_promote::type; + +// helper type for validating format strings +enum class JanssonFormatArg { + KEY, + STRING, + SIZE, + INT, + JSON_INT, + DOUBLE, + JSON, +}; + +// make a JanssonFormatArg from a type using template specializations +template JanssonFormatArg makeJanssonFormatArg() = delete; + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::KEY; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::STRING; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::SIZE; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::INT; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::INT; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::JSON_INT; +} + +template <> +consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::JSON_INT; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::DOUBLE; +} + +template <> consteval JanssonFormatArg makeJanssonFormatArg() { + return JanssonFormatArg::JSON; +} + +template +constexpr static JanssonFormatArg janssonFormatArg = makeJanssonFormatArg(); + +// helper type for validating format strings with nested structures +enum class JanssonNestedStructure { + OBJECT, + ARRAY, +}; + +// compile time validator for json_unpack-style format strings +template class JanssonUnpackFormatString { +public: + consteval JanssonUnpackFormatString(char const *fmt) : fmt(fmt) { + validate(fmt); + } + + constexpr char const *c_str() const { return fmt; } + +private: + consteval static void validate(char const *fmt) { + constexpr auto ignored = std::string_view(" \t\n,:"); + + auto const args = + std::initializer_list{janssonFormatArg>...}; + auto const string = std::string_view(fmt); + auto const findNextToken = [&](std::size_t pos) { + if (pos == std::string_view::npos) + return std::string_view::npos; + + return string.find_first_not_of(ignored, pos); + }; + + std::size_t token = 0; + auto arg = args.begin(); + auto nested = std::vector{}; + + while ((token = findNextToken(token)) != std::string_view::npos) { + if (not nested.empty() and + nested.back() == JanssonNestedStructure::OBJECT) { + if (string[token] == '}') { + nested.pop_back(); + token++; + continue; + } + + if (string[token++] != 's') + throw "Invalid specifier for object key"; + + if (*arg++ != JanssonFormatArg::KEY) + throw "Expected 'const char *' argument for object key"; + + if ((token = findNextToken(token)) == std::string_view::npos) + break; + + if (string[token] == '?') + token++; + + if ((token = findNextToken(token)) == std::string_view::npos) + break; + } + + switch (string[token++]) { + case 's': { + if (*arg++ != JanssonFormatArg::STRING) + throw "Expected 'const char **' argument for 's' specifier"; + + if ((token = findNextToken(token)) == std::string_view::npos) + break; + + if (string[token] == '%') { + if (*arg++ != JanssonFormatArg::SIZE) + throw "Expected 'size_t *' argument for 's%' specifier"; + token++; + } + } break; + + case 'n': + break; + + case 'b': { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int *' argument for 'b' specifier"; + } break; + + case 'i': { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int *' argument for 'i' specifier"; + } break; + + case 'I': { + if (*arg++ != JanssonFormatArg::JSON_INT) + throw "Expected 'json_int_t *' argument for 'I' specifier"; + } break; + + case 'f': { + if (*arg++ != JanssonFormatArg::DOUBLE) + throw "Expected 'double *' argument for 'f' specifier"; + } break; + + case 'F': { + if (*arg++ != JanssonFormatArg::DOUBLE) + throw "Expected 'double *' argument for 'F' specifier"; + } break; + + case 'o': { + if (*arg++ != JanssonFormatArg::JSON) + throw "Expected 'json_t *' argument for 'o' specifier"; + } break; + + case 'O': { + if (*arg++ != JanssonFormatArg::JSON) + throw "Expected 'json_t *' argument for 'O' specifier"; + } break; + + case '[': { + nested.push_back(JanssonNestedStructure::ARRAY); + } break; + + case ']': { + if (nested.empty() or nested.back() != JanssonNestedStructure::ARRAY) + throw "Unexpected ]"; + nested.pop_back(); + } break; + + case '{': { + nested.push_back(JanssonNestedStructure::OBJECT); + } break; + + case '}': + throw "Unexpected }"; + + case '!': + break; + + case '*': + break; + + default: + throw "Unknown format specifier in format string"; + } + } + + if (not nested.empty()) + throw "Unclosed structure in format string"; + + if (arg != args.end()) + throw "Unused trailing arguments"; + } + + char const *fmt; +}; + +template +void janssonUnpack(::json_t *json, + JanssonUnpackFormatString...> fmt, + Args... args) { + ::json_error_t err; + if (auto ret = ::json_unpack_ex(json, &err, 0, fmt.c_str(), args...); + ret == -1) + throw RuntimeError("Could not unpack json value: {}", err.text); +} + +// compile time validator for json_pack-style format strings +template class JanssonPackFormatString { +public: + consteval JanssonPackFormatString(char const *fmt) : fmt(fmt) { + validate(fmt); + } + + constexpr char const *c_str() const { return fmt; } + +private: + consteval static void validate(char const *fmt) { + constexpr auto ignored = std::string_view(" \t\n,:"); + + auto const args = std::initializer_list{janssonFormatArg...}; + auto const string = std::string_view(fmt); + auto const findNextToken = [&](std::size_t pos) { + if (pos == std::string_view::npos) + return std::string_view::npos; + + return string.find_first_not_of(ignored, pos); + }; + + std::size_t token = 0; + auto arg = args.begin(); + auto nested = std::vector{}; + while ((token = findNextToken(token)) != std::string_view::npos) { + if (not nested.empty() and + nested.back() == JanssonNestedStructure::OBJECT) { + if (string[token] == '}') { + nested.pop_back(); + token++; + continue; + } + + if (string[token++] != 's') + throw "Invalid specifier for object key"; + + if (*arg++ != JanssonFormatArg::STRING) + throw "Expected 'const char *' argument for object key"; + + if ((token = findNextToken(token)) == std::string_view::npos) + break; + + if (string[token] == '#') { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for 's#' specifier"; + token++; + } else if (string[token] == '%') { + if (*arg++ != JanssonFormatArg::SIZE) + throw "Expected 'size_t' argument for 's%' specifier"; + token++; + } + + while ((token = findNextToken(token)) != std::string_view::npos) { + if (string[token] != '+') + break; + + if (*arg++ != JanssonFormatArg::STRING) + throw "Expected 'const char *' argument for '+' specifier"; + + if ((token = findNextToken(++token)) == std::string_view::npos) + break; + + if (string[token] == '#') { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for '+#' specifier"; + token++; + } else if (string[token] == '%') { + if (*arg++ != JanssonFormatArg::SIZE) + throw "Expected 'size_t' argument for 's%' specifier"; + token++; + } + } + + if (token == std::string_view::npos) + throw "Expected closing }"; + } + + switch (string[token++]) { + case 's': { + if (*arg++ != JanssonFormatArg::STRING) + throw "Expected 'const char *' argument for 's' specifier"; + + if ((token = findNextToken(token)) == std::string_view::npos) + break; + + auto optionalValue = false; + if (string[token] == '?' or string[token] == '*') { + optionalValue = true; + token++; + } else if (string[token] == '#') { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for 's#' specifier"; + token++; + } else if (string[token] == '%') { + if (*arg++ != JanssonFormatArg::SIZE) + throw "Expected 'size_t' argument for 's%' specifier"; + token++; + } + + if (optionalValue) + break; + + while ((token = findNextToken(token)) != std::string_view::npos) { + if (string[token] != '+') + break; + + if (*arg++ != JanssonFormatArg::STRING) + throw "Expected 'const char *' argument for '+' specifier"; + + if ((token = findNextToken(++token)) == std::string_view::npos) + break; + + if (string[token] == '#') { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for '+#' specifier"; + token++; + } else if (string[token] == '%') { + if (*arg++ != JanssonFormatArg::SIZE) + throw "Expected 'size_t' argument for 's%' specifier"; + token++; + } + } + } break; + + case '+': + throw "Unexpected '+' specifier"; + + case 'n': + break; + + case 'b': { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for 'b' specifier"; + } break; + + case 'i': { + if (*arg++ != JanssonFormatArg::INT) + throw "Expected 'int' argument for 'i' specifier"; + } break; + + case 'I': { + if (*arg++ != JanssonFormatArg::JSON_INT) + throw "Expected 'json_int_t' argument for 'I' specifier"; + } break; + + case 'f': { + if (*arg++ != JanssonFormatArg::DOUBLE) + throw "Expected 'double' argument for 'f' specifier"; + } break; + + case 'o': { + if (*arg++ != JanssonFormatArg::JSON) + throw "Expected 'json_t *' argument for 'o' specifier"; + + if (string[token] == '?' or string[token] == '*') + token++; + } break; + + case 'O': { + if (*arg++ != JanssonFormatArg::JSON) + throw "Expected 'json_t *' argument for 'O' specifier"; + + if (string[token] == '?' or string[token] == '*') + token++; + } break; + + case '[': { + nested.push_back(JanssonNestedStructure::ARRAY); + } break; + + case ']': { + if (nested.empty() or nested.back() != JanssonNestedStructure::ARRAY) + throw "Unexpected ]"; + nested.pop_back(); + } break; + + case '{': { + nested.push_back(JanssonNestedStructure::OBJECT); + } break; + + case '}': + throw "Unexpected }"; + + default: + throw "Unknown format specifier in format string"; + } + } + + if (not nested.empty()) + throw "Unclosed structure in format string"; + + if (arg != args.end()) + throw "Unused trailing arguments"; + } + + char const *fmt; +}; + +template +JanssonPtr +janssonPack(JanssonPackFormatString...> fmt, + Args... args) { + auto json = ::json_pack(fmt.c_str(), args...); + if (json == nullptr) + throw RuntimeError("Could not pack json value"); + + return JanssonPtr(json); +} + +} // namespace villas diff --git a/common/include/villas/json.hpp b/common/include/villas/json.hpp new file mode 100644 index 000000000..673ad83b9 --- /dev/null +++ b/common/include/villas/json.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +#include + +namespace villas { + +using Json = nlohmann::json; +using JsonPointer = nlohmann::json_pointer; + +// forward declaration for villas/jansson.hpp compatibility header +class JanssonPtr; +void to_json(Json &json, JanssonPtr const &jansson); +void from_json(Json const &json, JanssonPtr &jansson); + +// load a configuration file +Json load_config_deprecated(fs::path const &path, bool resolve_inc, + bool resolve_env); + +}; // namespace villas + +// forward declaration for libjansson's json_t +struct json_t; +void to_json(villas::Json &json, json_t const *jansson); + +template <> // format config_json using operator<< +struct fmt::formatter : ostream_formatter {}; + +template <> // format config_json_pointer using operator<< +struct fmt::formatter : ostream_formatter {}; diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index 318aeceac..0919013bf 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -131,6 +131,10 @@ template struct overloaded : Ts... { // Explicit deduction guide (not needed as of C++20) template overloaded(Ts...) -> overloaded; +// glob-style filesystem pattern matching +std::vector glob(fs::path const &pattern, + std::span searchDirectories); + void write_to_file(std::string data, const fs::path file); namespace base64 { diff --git a/common/lib/CMakeLists.txt b/common/lib/CMakeLists.txt index 497fe9994..816df3a42 100644 --- a/common/lib/CMakeLists.txt +++ b/common/lib/CMakeLists.txt @@ -15,6 +15,7 @@ add_library(villas-common SHARED cpuset.cpp dsp/pid.cpp hist.cpp + json.cpp kernel/kernel.cpp kernel/rt.cpp list.cpp @@ -55,6 +56,7 @@ target_include_directories(villas-common PUBLIC ) target_link_libraries(villas-common PUBLIC + nlohmann_json::nlohmann_json PkgConfig::JANSSON PkgConfig::UUID ${OPENSSL_LIBRARIES} diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index de1a2278e..dd2f7f0ab 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -203,15 +204,24 @@ char *Hist::dump() const { json_t *Hist::toJson() const { json_t *json_buckets, *json_hist; - json_hist = json_pack("{ s: f, s: f, s: i }", "low", low, "high", high, - "total", total); + json_hist = janssonPack("{ s:f, s:f, s:I }", // + "low", low, // + "high", high, // + "total", static_cast(total)) + .release(); if (total > 0) { - json_object_update(json_hist, - json_pack("{ s: i, s: i, s: f, s: f, s: f, s: f, s: f }", - "higher", higher, "lower", lower, "highest", - highest, "lowest", lowest, "mean", getMean(), - "variance", getVar(), "stddev", getStddev())); + json_object_update_new( + json_hist, + janssonPack("{ s:I, s:I, s:f, s:f, s:f, s:f, s:f }", // + "higher", static_cast(higher), // + "lower", static_cast(lower), // + "highest", highest, // + "lowest", lowest, // + "mean", getMean(), // + "variance", getVar(), // + "stddev", getStddev()) + .release()); } if (total - lower - higher > 0) { @@ -220,7 +230,7 @@ json_t *Hist::toJson() const { for (auto elm : data) json_array_append(json_buckets, json_integer(elm)); - json_object_set(json_hist, "buckets", json_buckets); + json_object_set_new(json_hist, "buckets", json_buckets); } return json_hist; diff --git a/common/lib/json.cpp b/common/lib/json.cpp new file mode 100644 index 000000000..126fbde6d --- /dev/null +++ b/common/lib/json.cpp @@ -0,0 +1,370 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace std::string_view_literals; + +void to_json(villas::Json &json, ::json_t const *jansson) { + switch (json_typeof(jansson)) { + using enum json_type; + + case JSON_ARRAY: { + std::size_t index; + ::json_t const *value; + + json = villas::Json::array(); + json_array_foreach (jansson, index, value) + json.push_back(value); + } break; + + case JSON_OBJECT: { + char const *key; + std::size_t keylen; + ::json_t const *value; + + json = villas::Json::object(); + // The const_cast below is safe as long as we don't replace the contained value + // by accessing the underlying iterator with json_object_key_to_iter(key) and + // json_object_iter_set or json_object_iter_set_new. + // + // There is no API in jansson value to iterate or even enumerate keys of a const object. + // + // See https://github.com/akheron/jansson/issues/578 + json_object_keylen_foreach (const_cast<::json_t *>(jansson), key, keylen, + value) + json.emplace(std::string{key, keylen}, value); + } break; + + case JSON_STRING: { + json = std::string{json_string_value(jansson), json_string_length(jansson)}; + } break; + + case JSON_INTEGER: { + json = json_integer_value(jansson); + } break; + + case JSON_REAL: { + json = json_real_value(jansson); + } break; + + case JSON_TRUE: { + json = true; + } break; + + case JSON_FALSE: { + json = false; + } break; + + case JSON_NULL: + default: { + json = nullptr; + } break; + } +} + +namespace villas { + +void from_json(Json const &json, JanssonPtr &jansson) { + switch (json.type()) { + using value_t = Json::value_t; + + case value_t::array: { + jansson.reset(json_array()); + for (auto const &item : json) + json_array_append_new(jansson.get(), item.get().release()); + } break; + + case value_t::object: { + jansson.reset(json_object()); + for (auto const &[key, value] : json.items()) + json_object_setn_new_nocheck(jansson.get(), key.data(), key.size(), + value.get().release()); + } break; + + case value_t::string: { + auto const string = json.get(); + jansson.reset(json_stringn_nocheck(string->data(), string->size())); + } break; + + case value_t::number_integer: { + auto const integer = json.get(); + jansson.reset(json_integer(*integer)); + } break; + + case value_t::number_unsigned: { + auto const integer = json.get(); + jansson.reset(json_integer(*integer)); + } break; + + case value_t::number_float: { + auto const real = json.get(); + jansson.reset(json_real(*real)); + } break; + + case value_t::boolean: { + auto const boolean = json.get(); + jansson.reset(json_boolean(*boolean)); + } break; + + case value_t::null: { + jansson.reset(json_null()); + } break; + + case value_t::binary: + throw std::runtime_error{"cannot convert binary value to jansson"}; + + case value_t::discarded: + throw std::runtime_error{"cannot convert discarded value to jansson"}; + + default: + __builtin_unreachable(); + } +} + +void to_json(Json &json, JanssonPtr const &jansson) { + to_json(json, jansson.get()); +} + +namespace { + +// implementation of deprecated variable substitutions +void expand_substitutions_deprecated(Json &value, bool resolve_env, + fs::path const *include_dir) { + if (not value.is_string()) + return; + + if (not resolve_env and not include_dir) + return; + + constexpr static auto DEPRECATED_INCLUDE_KEYWORD = "@include "sv; + auto logger = Log::get("config"); + auto string = value.get_ref(); + auto do_include = false; + auto expanded = std::size_t{0}; + + // check for legacy @include keyword + if (include_dir and string.starts_with(DEPRECATED_INCLUDE_KEYWORD)) { + do_include = true; + expanded = DEPRECATED_INCLUDE_KEYWORD.length(); + } + + // legacy environment variable substitution syntax + static auto const env_regex = std::regex(R"--(\$\{([^}]+)\})--"); + enum : std::size_t { + CAPTURE_ALL = 0, + CAPTURE_NAME, + }; + + // expand deprecated environment substition syntax + std::smatch match; + while (resolve_env and std::regex_search(string.cbegin() + expanded, + string.cend(), match, env_regex)) { + auto name = std::string(match[CAPTURE_NAME]); + auto env_ptr = std::getenv(name.c_str()); + if (not env_ptr) + throw std::runtime_error( + fmt::format("Could substitute environment variable {:?}", name)); + + auto env_value = std::string_view(env_ptr); + auto [begin, end] = std::pair(match[CAPTURE_ALL]); + string.replace(begin, end, env_value.begin(), env_value.end()); + expanded += match.position() + env_value.length(); + } + + // expand deprecated @include directive + if (do_include) { + auto pattern = + std::string_view(string).substr(DEPRECATED_INCLUDE_KEYWORD.length()); + auto result = Json(nullptr); + for (auto path : utils::glob(pattern, std::span(include_dir, 1))) { + auto partial_result = + load_config_deprecated(path, include_dir != nullptr, resolve_env); + if (result.is_null()) + result = partial_result; + else if (partial_result.is_object() and result.is_object()) + result.update(partial_result, true); + else if (partial_result.is_array() and result.is_array()) + result.insert(result.end(), partial_result.begin(), + partial_result.end()); + } + + logger->warn("Found deprecated @include directive: {}", value); + value = std::move(result); + } else if (expanded) { + logger->warn("Found deprecated environment substitution: {}", value); + value = std::move(string); + } +} + +Json parse_libconfig_setting(::config_setting_t const *setting, + bool resolve_env, fs::path const *include_dir) { + auto logger = Log::get("config"); + + switch (config_setting_type(setting)) { + case CONFIG_TYPE_ARRAY: + case CONFIG_TYPE_LIST: { + auto array = Json::array(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + array.push_back(parse_libconfig_setting(elem, resolve_env, include_dir)); + } + + return array; + } + + case CONFIG_TYPE_GROUP: { + auto object = Json::object(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + auto name = std::string(config_setting_name(elem)); + object.emplace(std::move(name), + parse_libconfig_setting(elem, resolve_env, include_dir)); + } + + return object; + } + + case CONFIG_TYPE_STRING: { + auto json = Json(std::string(config_setting_get_string(setting))); + expand_substitutions_deprecated(json, resolve_env, include_dir); + return json; + } + + case CONFIG_TYPE_INT: { + return config_setting_get_int(setting); + } + + case CONFIG_TYPE_INT64: { + return std::int64_t{config_setting_get_int64(setting)}; + } + + case CONFIG_TYPE_FLOAT: { + return config_setting_get_float(setting); + } + + case CONFIG_TYPE_BOOL: { + return static_cast(config_setting_get_bool(setting)); + } + + case CONFIG_TYPE_NONE: + default: { + return nullptr; + } + } +} + +struct LibconfigHook { + bool resolve_env; + fs::path const *include_dir; +}; + +extern "C" char const **libconfig_include_func(::config_t *config, char const *, + char const *pattern, + char const **error) noexcept { + auto hook = static_cast(config_get_hook(config)); + auto paths = std::vector{}; + + if (not hook->include_dir) { + *error = "include directives are disabled"; + return nullptr; + } + + auto pattern_json = Json(pattern); + expand_substitutions_deprecated(pattern_json, hook->resolve_env, nullptr); + auto const &pattern_expanded = pattern_json.get_ref(); + + try { + paths = utils::glob(pattern_expanded, std::span(hook->include_dir, 1)); + std::erase_if(paths, [](auto const &path) { + auto ec = std::error_code{}; + return not fs::is_regular_file(path, ec); + }); + } catch (...) { + } + + if (paths.empty()) { + *error = "include directive did not match any file"; + return nullptr; + } + + auto ret = + static_cast(std::calloc(paths.size() + 1, sizeof(char *))); + auto index = std::size_t{0}; + for (auto &path : paths) + ret[index++] = strdup(path.c_str()); + + return ret; +} + +Json load_libconfig_deprecated(std::FILE *file, bool resolve_env, + fs::path const *include_dir) { + using ConfigDeleter = decltype([](::config_t *c) { ::config_destroy(c); }); + using ConfigPtr = std::unique_ptr<::config_t, ConfigDeleter>; + + auto hook = LibconfigHook{ + .resolve_env = resolve_env, + .include_dir = include_dir, + }; + + ::config_t config; + ::config_init(&config); + auto guard = ConfigPtr(&config); + + ::config_set_hook(&config, &hook); + ::config_set_include_func(&config, &libconfig_include_func); + if (auto ret = ::config_read(&config, file); ret != CONFIG_TRUE) { + throw std::runtime_error(fmt::format("Failed to load libconfig file: {}", + config_error_text(&config))); + } + + return parse_libconfig_setting(config_root_setting(&config), resolve_env, + include_dir); +} + +} // namespace + +Json load_config_deprecated(fs::path const &path, bool resolve_inc, + bool resolve_env) { + using FileDeleter = decltype([](std::FILE *c) { std::fclose(c); }); + using FilePtr = std::unique_ptr; + + auto file = FilePtr(std::fopen(path.c_str(), "r")); + if (not file) { + throw std::runtime_error( + fmt::format("Failed to open config file {}", path.string())); + } + + auto include_dir = path.parent_path(); + if (auto ext = path.extension(); ext == ".json") { + auto parser_callback = [&](int depth, Json::parse_event_t event, + Json &value) { + if (event == Json::parse_event_t::value) + expand_substitutions_deprecated(value, resolve_env, + resolve_inc ? &include_dir : nullptr); + + return true; + }; + + return Json::parse(file.get(), parser_callback); + } else if (ext == ".conf") { + return load_libconfig_deprecated(file.get(), resolve_env, + resolve_inc ? &include_dir : nullptr); + } else { + throw std::runtime_error(fmt::format( + "Failed to load config file with unknown extension {}", ext.string())); + } +} + +} // namespace villas diff --git a/common/lib/log.cpp b/common/lib/log.cpp index e9c653517..cddc7b589 100644 --- a/common/lib/log.cpp +++ b/common/lib/log.cpp @@ -6,7 +6,6 @@ */ #include -#include #include #include @@ -14,6 +13,7 @@ #include #include +#include #include #include @@ -100,17 +100,15 @@ void Log::parse(json_t *json) { const char *path = nullptr; const char *pattern = nullptr; - int ret, syslog = 0; - - json_error_t err; + int syslog = 0; json_t *json_expressions = nullptr; - ret = json_unpack_ex(json, &err, JSON_STRICT, - "{ s?: s, s?: s, s?: o, s?: b, s?: s }", "level", &level, - "file", &path, "expressions", &json_expressions, - "syslog", &syslog, "pattern", &pattern); - if (ret) - throw ConfigError(json, err, "node-config-logging"); + janssonUnpack(json, "{ s?: s, s?: s, s?: o, s?: b, s?: s }", // + "level", &level, // + "file", &path, // + "expressions", &json_expressions, // + "syslog", &syslog, // + "pattern", &pattern); if (level) setLevel(level); @@ -179,22 +177,16 @@ Log::Level Log::getLevel() const { return level; } std::string Log::getLevelName() const { auto sv = spdlog::level::to_string_view(level); - return std::string(sv.data()); + return std::string(sv.begin(), sv.end()); } Log::Expression::Expression(json_t *json) { - int ret; - const char *nme; const char *lvl; - json_error_t err; - - ret = json_unpack_ex(json, &err, JSON_STRICT, "{ s: s, s: s }", "name", &nme, - "level", &lvl); - if (ret) - throw ConfigError(json, err, "node-config-logging-expressions"); - + janssonUnpack(json, "{ s: s, s: s }", // + "name", &nme, // + "level", &lvl); level = spdlog::level::from_str(lvl); name = nme; } diff --git a/common/lib/utils.cpp b/common/lib/utils.cpp index 2bbeda096..618f45a7e 100644 --- a/common/lib/utils.cpp +++ b/common/lib/utils.cpp @@ -12,12 +12,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -352,6 +354,70 @@ bool isPrivileged() { return true; } +// internal glob implementation details +namespace { +bool isGlobPattern(fs::path const &path) { + static const auto specialCharacters = fs::path("?*[").native(); + auto const &string = path.native(); + return std::ranges::find_first_of(string, specialCharacters) != string.end(); +} + +bool isGlobMatch(fs::path const &pattern, fs::path const &path) { + return ::fnmatch(pattern.c_str(), path.c_str(), FNM_PATHNAME) == 0; +} + +void globImpl(std::vector &result, fs::path &&path, + std::ranges::subrange pattern) { + [[maybe_unused]] auto discardErrorCode = std::error_code{}; + + if (pattern.empty()) { + // we've reached the end of our pattern + if (fs::exists(path, discardErrorCode)) + result.push_back(path); + return; + } + + if (not fs::is_directory(path, discardErrorCode)) + return; + + if (not isGlobPattern(pattern.front())) { + path /= pattern.front(); + return globImpl(result, std::move(path), std::move(pattern).next()); + } else { + auto nextPattern = pattern.next(); + for (auto entry : fs::directory_iterator(path)) { + if (not isGlobMatch(pattern.front(), entry.path().filename())) + continue; + + globImpl(result, fs::path(entry.path()), nextPattern); + } + } +} +} // namespace + +std::vector glob(fs::path const &pattern, + std::span searchDirectories) { + auto logger = Log::get("glob"); + std::vector result; + if (pattern.is_absolute()) { + logger->debug("Matching absolute pattern {:?}", pattern.string()); + globImpl(result, pattern.root_path(), pattern); + } else { + for (auto path : searchDirectories) { + logger->debug("Matching relative pattern {:?} in {:?}", pattern.string(), + path.string()); + globImpl(result, std::move(path), pattern); + } + } + + if (result.empty()) { + throw std::runtime_error( + fmt::format("Could not find any file matching {:?}", pattern.string())); + } + + return result; +} + void write_to_file(std::string data, const fs::path file) { villas::Log::get("Filewriter")->debug("{} > {}", data, file.string()); std::ofstream outputFile(file.string()); diff --git a/common/tests/unit/buffer.cpp b/common/tests/unit/buffer.cpp index f6043c0d8..f4451724a 100644 --- a/common/tests/unit/buffer.cpp +++ b/common/tests/unit/buffer.cpp @@ -9,9 +9,9 @@ #include #include -#include #include +#include using namespace villas; @@ -95,7 +95,7 @@ Test(buffer, multiple) { std::srand(std::time(nullptr)); for (int i = 0; i < N; i++) { - k[i] = json_pack("{ s: i }", "id", std::rand()); + k[i] = janssonPack("{ s:i }", "id", std::rand()).release(); cr_assert_not_null(k[i]); ret = buf.encode(k[i]); diff --git a/include/villas/config_class.hpp b/include/villas/config_class.hpp deleted file mode 100644 index e6b372b9c..000000000 --- a/include/villas/config_class.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* Configuration file parsing. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include - -#include -#include - -#include -#include -#include - -#ifdef WITH_CONFIG -#include -#endif - -namespace villas { -namespace node { - -class Config { - -protected: - using str_walk_fcn_t = std::function; - - Logger logger; - - std::list includeDirectories; - std::string configPath; - - // Check if file exists on local system. - static bool isLocalFile(const std::string &uri) { - return access(uri.c_str(), F_OK) != -1; - } - - // Decode configuration file. - json_t *decode(FILE *f); - -#ifdef WITH_CONFIG - // Convert libconfig .conf file to libjansson .json file. - json_t *libconfigDecode(FILE *f); - - static const char **includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error); - - const char **includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error); -#endif // WITH_CONFIG - - // Load configuration from standard input (stdim). - FILE *loadFromStdio(); - - // Load configuration from local file. - FILE *loadFromLocalFile(const std::string &u); - - std::list resolveIncludes(const std::string &name); - - void resolveEnvVars(std::string &text); - - // Resolve custom include directives. - json_t *expandIncludes(json_t *in); - - // To shell-like subsitution of environment variables in strings. - json_t *expandEnvVars(json_t *in); - - // Run a callback function for each string in the config - json_t *walkStrings(json_t *in, str_walk_fcn_t cb); - - // Get the include dirs - std::list getIncludeDirectories(FILE *f) const; - -public: - json_t *root; - - Config(); - Config(const std::string &u); - - ~Config(); - - json_t *load(std::FILE *f, bool resolveIncludes = true, - bool resolveEnvVars = true); - - json_t *load(const std::string &u, bool resolveIncludes = true, - bool resolveEnvVars = true); - - std::string const &getConfigPath() const { return configPath; } -}; - -} // namespace node -} // namespace villas diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index f4836fdfd..7f197ebfb 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -15,11 +15,10 @@ extern "C" { } #endif -#include - #include #include -#include +#include +#include #include #include #include @@ -68,7 +67,8 @@ class SuperNode { struct timespec started; // The time at which the instance has been started. - Config config; // The configuration file. + fs::path configPath; + JanssonPtr configRoot; // The configuration file. public: // Inititalize configuration object before parsing the configuration. @@ -77,7 +77,7 @@ class SuperNode { int init(); // Wrapper for parse() which loads the config first. - void parse(const std::string &name); + void parse(fs::path const &path); /* Parse super-node configuration. * @@ -138,9 +138,9 @@ class SuperNode { Web *getWeb() { return &web; } #endif - json_t *getConfig() { return config.root; } + json_t *getConfig() { return configRoot.get(); } - const std::string &getConfigPath() const { return config.getConfigPath(); } + fs::path const &getConfigPath() const { return configPath; } int getAffinity() const { return affinity; } diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e95215e5a..e0e904018 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -23,7 +23,6 @@ set(LIBRARIES set(LIB_SRC capabilities.cpp config_helper.cpp - config.cpp dumper.cpp format.cpp mapping.cpp diff --git a/lib/config.cpp b/lib/config.cpp deleted file mode 100644 index 0bab007ef..000000000 --- a/lib/config.cpp +++ /dev/null @@ -1,368 +0,0 @@ -/* Configuration file parsing. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#ifdef WITH_CONFIG -#include -#endif - -using namespace villas; -using namespace villas::node; - -Config::Config() : logger(Log::get("config")), root(nullptr) {} - -Config::Config(const std::string &u) : Config() { root = load(u); } - -Config::~Config() { json_decref(root); } - -json_t *Config::load(std::FILE *f, bool resolveInc, bool resolveEnvVars) { - json_t *root = decode(f); - - if (resolveInc) { - json_t *root_old = root; - root = expandIncludes(root); - json_decref(root_old); - } - - if (resolveEnvVars) { - json_t *root_old = root; - root = expandEnvVars(root); - json_decref(root_old); - } - - return root; -} - -json_t *Config::load(const std::string &u, bool resolveInc, - bool resolveEnvVars) { - FILE *f; - - if (u == "-") - f = loadFromStdio(); - else - f = loadFromLocalFile(u); - - json_t *root = load(f, resolveInc, resolveEnvVars); - - fclose(f); - - return root; -} - -FILE *Config::loadFromStdio() { - logger->info("Reading configuration from standard input"); - - auto *cwd = new char[PATH_MAX]; - - configPath = getcwd(cwd, PATH_MAX); - - delete[] cwd; - - return stdin; -} - -FILE *Config::loadFromLocalFile(const std::string &u) { - logger->info("Reading configuration from local file: {}", u); - - configPath = u; - FILE *f = fopen(u.c_str(), "r"); - if (!f) - throw RuntimeError("Failed to open configuration from: {}", u); - - return f; -} - -json_t *Config::decode(FILE *f) { - json_error_t err; - - // Update list of include directories - auto incDirs = getIncludeDirectories(f); - includeDirectories.insert(includeDirectories.end(), incDirs.begin(), - incDirs.end()); - - json_t *root = json_loadf(f, 0, &err); - if (root == nullptr) { -#ifdef WITH_CONFIG - // We try again to parse the config in the legacy format - root = libconfigDecode(f); -#else - throw JanssonParseError(err); -#endif // WITH_CONFIG - } - - return root; -} - -std::list Config::getIncludeDirectories(FILE *f) const { - int ret, fd; - char buf[PATH_MAX]; - char *dir; - - std::list dirs; - - // Adding directory of base configuration file - fd = fileno(f); - if (fd < 0) - throw SystemError("Failed to get file descriptor"); - - auto path = fmt::format("/proc/self/fd/{}", fd); - - ret = readlink(path.c_str(), buf, sizeof(buf)); - if (ret > 0) { - buf[ret] = 0; - if (isLocalFile(buf)) { - dir = dirname(buf); - dirs.push_back(dir); - } - } - - // Adding current working directory - dir = getcwd(buf, sizeof(buf)); - if (dir != nullptr) - dirs.push_back(dir); - - return dirs; -} - -std::list Config::resolveIncludes(const std::string &n) { - glob_t gb; - int ret, flags = 0; - - memset(&gb, 0, sizeof(gb)); - - auto name = n; - resolveEnvVars(name); - - if (name.size() >= 1 && name[0] == '/') { // absolute path - ret = glob(name.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) - gb.gl_pathc = 0; - } else { // relative path - for (auto &dir : includeDirectories) { - auto pattern = fmt::format("{}/{}", dir, name.c_str()); - - ret = glob(pattern.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) { - gb.gl_pathc = 0; - - goto out; - } - - flags |= GLOB_APPEND; - } - } - -out: - std::list files; - for (unsigned i = 0; i < gb.gl_pathc; i++) - files.push_back(gb.gl_pathv[i]); - - globfree(&gb); - - return files; -} - -void Config::resolveEnvVars(std::string &text) { - static const std::regex env_re{R"--(\$\{([^}]+)\})--"}; - - std::smatch match; - while (std::regex_search(text, match, env_re)) { - auto const from = match[0]; - auto const var_name = match[1].str(); - char *var_value = std::getenv(var_name.c_str()); - if (!var_value) - throw RuntimeError("Unresolved environment variable: {}", var_name); - - text.replace(from.first - text.begin(), from.second - from.first, - var_value); - - logger->debug("Replace env var {} in \"{}\" with value \"{}\"", var_name, - text, var_value); - } -} - -#ifdef WITH_CONFIG -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) -const char **Config::includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - void *ctx = config_get_hook(cfg); - - return reinterpret_cast(ctx)->includeFunc(cfg, include_dir, path, - error); -} - -const char **Config::includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - auto paths = resolveIncludes(path); - - unsigned i = 0; - auto files = (const char **)malloc(sizeof(char **) * (paths.size() + 1)); - - for (auto &path : paths) - files[i++] = strdup(path.c_str()); - - files[i] = NULL; - - return files; -} -#endif - -json_t *Config::libconfigDecode(FILE *f) { - int ret; - - config_t cfg; - config_setting_t *cfg_root; - config_init(&cfg); - config_set_auto_convert(&cfg, 1); - - // Setup libconfig include path -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) - config_set_hook(&cfg, this); - - config_set_include_func(&cfg, includeFuncStub); -#else - if (includeDirectories.size() > 0) { - logger->info("Setting include dir to: {}", includeDirectories.front()); - - config_set_include_dir(&cfg, includeDirectories.front().c_str()); - - if (includeDirectories.size() > 1) { - logger->warn( - "Ignoring all but the first include directories for libconfig"); - logger->warn( - " libconfig does not support more than a single include dir!"); - } - } -#endif - - // Rewind before re-reading - rewind(f); - - ret = config_read(&cfg, f); - if (ret != CONFIG_TRUE) - throw LibconfigParseError(&cfg); - - cfg_root = config_root_setting(&cfg); - - json_t *root = config_to_json(cfg_root); - if (!root) - throw RuntimeError("Failed to convert JSON to configuration file"); - - config_destroy(&cfg); - - return root; -} -#endif // WITH_CONFIG - -json_t *Config::walkStrings(json_t *root, str_walk_fcn_t cb) { - const char *key; - size_t index; - json_t *val, *new_val, *new_root; - - switch (json_typeof(root)) { - case JSON_STRING: - return cb(root); - - case JSON_OBJECT: - new_root = json_object(); - - json_object_foreach (root, key, val) { - new_val = walkStrings(val, cb); - - json_object_set_new(new_root, key, new_val); - } - - return new_root; - - case JSON_ARRAY: - new_root = json_array(); - - json_array_foreach (root, index, val) { - new_val = walkStrings(val, cb); - - json_array_append_new(new_root, new_val); - } - - return new_root; - - default: - return json_incref(root); - }; -} - -json_t *Config::expandEnvVars(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - std::string text = json_string_value(str); - - resolveEnvVars(text); - - return json_string(text.c_str()); - }); -} - -json_t *Config::expandIncludes(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - int ret; - std::string text = json_string_value(str); - static const std::string kw = "@include "; - - auto res = std::mismatch(kw.begin(), kw.end(), text.begin()); - if (res.first != kw.end()) - return json_incref(str); - else { - std::string pattern = text.substr(kw.size()); - - resolveEnvVars(pattern); - - json_t *incl = nullptr; - - for (auto &path : resolveIncludes(pattern)) { - json_t *other = load(path); - if (!other) - throw ConfigError(str, "include", - "Failed to include config file from {}", path); - - if (!incl) - incl = other; - else if (json_is_object(incl) && json_is_object(other)) { - ret = json_object_update_recursive(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } else if (json_is_array(incl) && json_is_array(other)) { - ret = json_array_extend(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } - - logger->debug("Included config from: {}", path); - } - - return incl; - } - }); -} diff --git a/lib/node_direction.cpp b/lib/node_direction.cpp index c766eaae1..b064bcfc3 100644 --- a/lib/node_direction.cpp +++ b/lib/node_direction.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -24,19 +25,17 @@ NodeDirection::NodeDirection(enum NodeDirection::Direction dir, Node *n) int NodeDirection::parse(json_t *json) { int ret; - - json_error_t err; json_t *json_hooks = nullptr; json_t *json_signals = nullptr; config = json; - ret = json_unpack_ex(json, &err, 0, "{ s?: o, s?: o, s?: i, s?: b, s?: b }", - "hooks", &json_hooks, "signals", &json_signals, - "vectorize", &vectorize, "builtin", &builtin, "enabled", - &enabled); - if (ret) - throw ConfigError(json, err, "node-config-node-in"); + janssonUnpack(json, "{ s?o, s?o, s?i, s?b, s?b }", // + "hooks", &json_hooks, // + "signals", &json_signals, // + "vectorize", &vectorize, // + "builtin", &builtin, // + "enabled", &enabled); if (node->getFactory()->getFlags() & (int)NodeFactory::Flags::PROVIDES_SIGNALS) { @@ -53,10 +52,7 @@ int NodeDirection::parse(json_t *json) { json_t *json_name, *json_signal = json_signals; int count; - ret = json_unpack_ex(json_signal, &err, 0, "{ s: i }", "count", &count); - if (ret) - throw ConfigError(json_signals, "node-config-node-signals", - "Invalid signal definition"); + janssonUnpack(json_signal, "{ s:i }", "count", &count); json_signals = json_array(); for (int i = 0; i < count; i++) { diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp index 4d8ccc3f3..e4af36d3b 100644 --- a/lib/nodes/fpga.cpp +++ b/lib/nodes/fpga.cpp @@ -381,8 +381,7 @@ int FpgaNodeFactory::start(SuperNode *sn) { } if (cards.empty()) { - auto searchPath = - sn->getConfigPath().substr(0, sn->getConfigPath().rfind("/")); + auto searchPath = sn->getConfigPath(); createCards(sn->getConfig(), cards, searchPath, vfioContainer); } diff --git a/lib/nodes/webrtc.cpp b/lib/nodes/webrtc.cpp index 1ae565473..85a75fba1 100644 --- a/lib/nodes/webrtc.cpp +++ b/lib/nodes/webrtc.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -57,17 +58,22 @@ int WebRTCNode::parse(json_t *json) { const char *pr = nullptr; int ord = -1; int &rexmit = dci.reliability.rexmit.emplace(0); - json_t *json_ice = nullptr; + json_t *json_servers = nullptr; + int tcp = -1; json_t *json_format = nullptr; - json_error_t err; - ret = json_unpack_ex( - json, &err, 0, "{ s: s, s?: s, s?: s, s?: i, s?: i, s?: b, s?: o }", - "session", &sess, "peer", &pr, "server", &svr, "wait_seconds", - &wait_seconds, "max_retransmits", &rexmit, "ordered", &ord, "ice", - &json_ice, "format", &json_format); - if (ret) - throw ConfigError(json, err, "node-config-node-webrtc"); + janssonUnpack(json, + "{ s:s, s?s, s?s, s?i, s?i, s?b, s?{ s?o, s?b }, s?o }", // + "session", &sess, // + "peer", &pr, // + "server", &svr, // + "wait_seconds", &wait_seconds, // + "max_retransmits", &rexmit, // + "ordered", &ord, // + "ice", // + /* ice */ "servers", &json_servers, // + /* ice */ "tcp", &tcp, // + "format", &json_format); session = sess; @@ -80,39 +86,28 @@ int WebRTCNode::parse(json_t *json) { if (ord) dci.reliability.unordered = !ord; - if (json_ice) { - json_t *json_servers = nullptr; - - int tcp = -1; - - ret = json_unpack_ex(json_ice, &err, 0, "{ s?: o, s?: b }", "servers", - &json_servers, "tcp", &tcp); - if (ret) - throw ConfigError(json, err, "node-config-node-webrtc-ice"); + if (json_servers) { + rtcConf.iceServers.clear(); - if (json_servers) { - rtcConf.iceServers.clear(); + if (!json_is_array(json_servers)) + throw ConfigError( + json_servers, "node-config-node-webrtc-ice-servers", + "ICE Servers must be a an array of server configurations."); - if (!json_is_array(json_servers)) - throw ConfigError( - json_servers, "node-config-node-webrtc-ice-servers", - "ICE Servers must be a an array of server configurations."); + size_t i; + json_t *json_server; + json_array_foreach (json_servers, i, json_server) { + if (!json_is_string(json_server)) + throw ConfigError(json_server, "node-config-node-webrtc-ice-server", + "ICE servers must be provided as STUN/TURN url."); - size_t i; - json_t *json_server; - json_array_foreach (json_servers, i, json_server) { - if (!json_is_string(json_server)) - throw ConfigError(json_server, "node-config-node-webrtc-ice-server", - "ICE servers must be provided as STUN/TURN url."); + std::string uri = json_string_value(json_server); - std::string uri = json_string_value(json_server); - - rtcConf.iceServers.emplace_back(uri); - } - - if (tcp > 0) - rtcConf.enableIceTcp = tcp > 0; + rtcConf.iceServers.emplace_back(uri); } + + if (tcp > 0) + rtcConf.enableIceTcp = tcp > 0; } auto *fmt = json_format ? FormatFactory::make(json_format) diff --git a/lib/super_node.cpp b/lib/super_node.cpp index 110d194c3..4ba712753 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -7,7 +7,8 @@ #include #include -#include + +#include #include #include @@ -22,6 +23,8 @@ #include #include +#include "villas/json.hpp" + #ifdef WITH_NETEM #include #endif @@ -60,10 +63,11 @@ SuperNode::SuperNode() logger = Log::get("super_node"); } -void SuperNode::parse(const std::string &u) { - config.root = config.load(u); +void SuperNode::parse(fs::path const &path) { + configPath = path; + load_config_deprecated(path, true, true).get_to(configRoot); - parse(config.root); + parse(configRoot.get()); } void SuperNode::parse(json_t *root) { diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix index 246150a67..51d3e982b 100644 --- a/packaging/nix/villas.nix +++ b/packaging/nix/villas.nix @@ -1,9 +1,13 @@ # SPDX-FileCopyrightText: 2023 OPAL-RT Germany GmbH # SPDX-License-Identifier: Apache-2.0 { + lib, + stdenv, + makeWrapper, # General configuration src, version, + system, withGpl ? true, withAllExtras ? false, withAllFormats ? false, @@ -41,23 +45,24 @@ bash, cmake, coreutils, + curl, + gnugrep, graphviz, + jansson, jq, - lib, - makeWrapper, + libuuid, + libwebsockets, + nlohmann_json, + openssl, pkg-config, - stdenv, - system, + spdlog, # Optional dependencies boxfort, comedilib, criterion, - curl, czmq, cyrus_sasl, ethercat, - gnugrep, - jansson, lib60870, libconfig, libdatachannel, @@ -70,14 +75,11 @@ libsodium, libuldaq, libusb1, - libuuid, - libwebsockets, libxml2, lua, mosquitto, nanomsg, opendssc, - openssl, orchestra, pcre2, pkgsBuildBuild, @@ -89,7 +91,6 @@ rdkafka, rdma-core, redis-plus-plus, - spdlog, linuxHeaders, }: stdenv.mkDerivation { @@ -190,8 +191,9 @@ stdenv.mkDerivation { ]; propagatedBuildInputs = [ - libuuid jansson + libuuid + nlohmann_json ] ++ lib.optionals withFormatProtobuf [ protobuf diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 777ea498d..0bf5002db 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -6,7 +6,7 @@ add_custom_target(run-integration-tests COMMAND - /bin/bash -o pipefail -c \" + /usr/bin/env bash -o pipefail -c \" SRCDIR=${PROJECT_SOURCE_DIR} BUILDDIR=${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/tools/integration-tests.sh 2>&1 | c++filt\" diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f9098e53e..7f9ddd71c 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(unit-tests PUBLIC add_custom_target(run-unit-tests COMMAND - /bin/bash -o pipefail -c \" + /usr/bin/env bash -o pipefail -c \" $ 2>&1 | c++filt\" DEPENDS unit-tests diff --git a/tests/unit/config.cpp b/tests/unit/config.cpp index 0ecd3a2fe..3e6774a07 100644 --- a/tests/unit/config.cpp +++ b/tests/unit/config.cpp @@ -9,68 +9,89 @@ #include #include +#include +#include -#include +#include +#include +#include #include -using namespace villas::node; +using namespace std::string_view_literals; +using FileDeleter = decltype([](std::FILE *f) { std::fclose(f); }); +using FilePtr = std::unique_ptr; + +constexpr auto fileNameTemplate = "villas.unit-test.XXXXXX.conf"sv; +constexpr auto fileNameSuffix = ".conf"sv; // cppcheck-suppress syntaxError Test(config, env) { - const char *cfg_f = "test = \"${MY_ENV_VAR}\"\n"; - - std::FILE *f = std::tmpfile(); - std::fputs(cfg_f, f); - std::rewind(f); - - auto c = Config(); - - char env[] = "MY_ENV_VAR=mobydick"; - putenv(env); - - auto *r = c.load(f); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "test"); - cr_assert_not_null(j); - - cr_assert(json_is_string(j)); - cr_assert_str_eq("mobydick", json_string_value(j)); + auto config_string = R"libconfig( + test = "${MY_ENV_VAR}" + )libconfig"; + + auto config_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto config_fd = + ::mkstemps(config_path_template.data(), fileNameSuffix.length()); + auto config_file = FilePtr(::fdopen(config_fd, "w")); + auto config_path = fs::path(config_path_template); + std::fputs(config_string, config_file.get()); + config_file.reset(); + + auto config = villas::load_config_deprecated(config_path, true, true) + .get(); + ::setenv("MY_ENV_VAR", "mobydick", true); + + auto *root = config.get(); + cr_assert_not_null(root); + + auto *string = json_object_get(root, "test"); + cr_assert_not_null(string); + cr_assert(json_is_string(string)); + cr_assert_str_eq("mobydick", json_string_value(string)); } Test(config, include) { - const char *cfg_f2 = "magic = 1234\n"; - - char f2_fn_tpl[] = "/tmp/villas.unit-test.XXXXXX"; - int f2_fd = mkstemp(f2_fn_tpl); - - std::FILE *f2 = fdopen(f2_fd, "w"); - std::fputs(cfg_f2, f2); - std::rewind(f2); - - auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn_tpl); - - std::FILE *f1 = std::tmpfile(); - std::fputs(cfg_f1.c_str(), f1); - std::rewind(f1); - - auto env = fmt::format("{}", f2_fn_tpl); - setenv("INCLUDE_FILE", env.c_str(), true); - - auto c = Config(); - - auto *r = c.load(f1); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "subval"); - cr_assert_not_null(j); - - auto *j2 = json_object_get(j, "magic"); - cr_assert_not_null(j2); - - cr_assert(json_is_integer(j2)); - cr_assert_eq(json_number_value(j2), 1234); - - std::fclose(f2); - std::remove(f2_fn_tpl); + auto incString = R"libconfig( + magic = 1234 + )libconfig"; + + auto inc_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto inc_fd = ::mkstemps(inc_path_template.data(), fileNameSuffix.length()); + cr_assert(inc_fd >= 0); + auto inc_file = FilePtr(::fdopen(inc_fd, "w")); + cr_assert_not_null(inc_file); + auto inc_path = fs::path(inc_path_template); + std::fputs(incString, inc_file.get()); + inc_file.reset(); + + auto config_string = fmt::format(R"libconfig( + subval = {{ @include "{}" }} + )libconfig", + inc_path.string()); + auto config_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto config_fd = + ::mkstemps(config_path_template.data(), fileNameSuffix.length()); + cr_assert(config_fd >= 0); + auto config_file = FilePtr(::fdopen(config_fd, "w")); + cr_assert_not_null(config_file); + auto config_path = fs::path(config_path_template); + std::fputs(config_string.c_str(), config_file.get()); + config_file.reset(); + + auto config = villas::load_config_deprecated(config_path, true, true) + .get(); + auto *root = config.get(); + cr_assert_not_null(root); + + auto *subval = json_object_get(root, "subval"); + cr_assert_not_null(subval); + + auto *magic = json_object_get(subval, "magic"); + cr_assert_not_null(magic); + cr_assert(json_is_integer(magic)); + cr_assert_eq(json_number_value(magic), 1234); }