Skip to content

Commit 99e1270

Browse files
liuzicheng1987liuzicheng1987
committed
Began implementing the Avro bindings
1 parent 716de7b commit 99e1270

File tree

15 files changed

+765
-1
lines changed

15 files changed

+765
-1
lines changed

‎CMakeLists.txt‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.23)
33
option(REFLECTCPP_BUILD_SHARED "Build shared library" ${BUILD_SHARED_LIBS})
44

55
option(REFLECTCPP_JSON "Enable JSON support" ON) # enabled by default
6+
option(REFLECTCPP_AVRO "Enable AVRO support" OFF)
67
option(REFLECTCPP_BSON "Enable BSON support" OFF)
78
option(REFLECTCPP_CBOR "Enable CBOR support" OFF)
89
option(REFLECTCPP_FLEXBUFFERS "Enable flexbuffers support" OFF)
@@ -19,6 +20,7 @@ option(REFLECTCPP_USE_BUNDLED_DEPENDENCIES "Use the bundled dependencies" ON)
1920
set(REFLECTCPP_USE_VCPKG_DEFAULT OFF)
2021
if(REFLECTCPP_BUILD_BENCHMARKS)
2122
set(REFLECTCPP_JSON ON CACHE BOOL "" FORCE)
23+
set(REFLECTCPP_AVRO ON CACHE BOOL "" FORCE)
2224
set(REFLECTCPP_BSON ON CACHE BOOL "" FORCE)
2325
set(REFLECTCPP_CBOR ON CACHE BOOL "" FORCE)
2426
set(REFLECTCPP_FLEXBUFFERS ON CACHE BOOL "" FORCE)
@@ -28,7 +30,7 @@ if(REFLECTCPP_BUILD_BENCHMARKS)
2830
set(REFLECTCPP_YAML ON CACHE BOOL "" FORCE)
2931
endif()
3032
if (REFLECTCPP_BUILD_TESTS OR REFLECTCPP_BUILD_BENCHMARKS OR
31-
(REFLECTCPP_JSON AND NOT REFLECTCPP_USE_BUNDLED_DEPENDENCIES) OR
33+
(REFLECTCPP_JSON AND NOT REFLECTCPP_USE_BUNDLED_DEPENDENCIES) OR REFLECTCPP_AVRO OR
3234
REFLECTCPP_BSON OR REFLECTCPP_CBOR OR REFLECTCPP_FLEXBUFFERS OR REFLECTCPP_MSGPACK OR REFLECTCPP_XML OR REFLECTCPP_TOML OR REFLECTCPP_YAML)
3335
# enable vcpkg per default if require features other than JSON
3436
set(REFLECTCPP_USE_VCPKG_DEFAULT ON)
@@ -95,6 +97,18 @@ if (REFLECTCPP_JSON)
9597
endif ()
9698
endif ()
9799

100+
if (REFLECTCPP_AVRO)
101+
list(APPEND REFLECT_CPP_SOURCES
102+
src/reflectcpp_avro.cpp
103+
)
104+
target_include_directories(reflectcpp SYSTEM PRIVATE "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include")
105+
if (MSVC)
106+
target_link_libraries(reflectcpp PRIVATE "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/avro${CMAKE_STATIC_LIBRARY_SUFFIX}")
107+
else ()
108+
target_link_libraries(reflectcpp PRIVATE "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/libavro${CMAKE_STATIC_LIBRARY_SUFFIX}")
109+
endif ()
110+
endif ()
111+
98112
if (REFLECTCPP_BSON)
99113
list(APPEND REFLECT_CPP_SOURCES
100114
src/reflectcpp_bson.cpp

‎include/rfl/avro/Parser.hpp‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#ifndef RFL_AVRO_PARSER_HPP_
2+
#define RFL_AVRO_PARSER_HPP_
3+
4+
#include "../Tuple.hpp"
5+
#include "../parsing/Parser.hpp"
6+
#include "Reader.hpp"
7+
#include "Writer.hpp"
8+
9+
namespace rfl {
10+
namespace parsing {
11+
12+
/// AVRO requires us to explicitly set the number of fields in advance. Because
13+
/// of that, we require all of the fields and then set them to nullptr, if
14+
/// necessary.
15+
template <class ProcessorsType, class... FieldTypes>
16+
requires AreReaderAndWriter<avro::Reader, avro::Writer,
17+
NamedTuple<FieldTypes...>>
18+
struct Parser<avro::Reader, avro::Writer, NamedTuple<FieldTypes...>,
19+
ProcessorsType>
20+
: public NamedTupleParser<
21+
avro::Reader, avro::Writer,
22+
/*_ignore_empty_containers=*/false,
23+
/*_all_required=*/true,
24+
/*_no_field_names=*/ProcessorsType::no_field_names_, ProcessorsType,
25+
FieldTypes...> {
26+
};
27+
28+
template <class ProcessorsType, class... Ts>
29+
requires AreReaderAndWriter<avro::Reader, avro::Writer, rfl::Tuple<Ts...>>
30+
struct Parser<avro::Reader, avro::Writer, rfl::Tuple<Ts...>, ProcessorsType>
31+
: public TupleParser<avro::Reader, avro::Writer,
32+
/*_ignore_empty_containers=*/false,
33+
/*_all_required=*/true, ProcessorsType,
34+
rfl::Tuple<Ts...>> {
35+
};
36+
37+
template <class ProcessorsType, class... Ts>
38+
requires AreReaderAndWriter<avro::Reader, avro::Writer, std::tuple<Ts...>>
39+
struct Parser<avro::Reader, avro::Writer, std::tuple<Ts...>, ProcessorsType>
40+
: public TupleParser<avro::Reader, avro::Writer,
41+
/*_ignore_empty_containers=*/false,
42+
/*_all_required=*/true, ProcessorsType,
43+
std::tuple<Ts...>> {
44+
};
45+
46+
} // namespace parsing
47+
} // namespace rfl
48+
49+
namespace rfl {
50+
namespace avro {
51+
52+
template <class T, class ProcessorsType>
53+
using Parser = parsing::Parser<Reader, Writer, T, ProcessorsType>;
54+
55+
}
56+
} // namespace rfl
57+
58+
#endif

‎include/rfl/avro/Reader.hpp‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#ifndef RFL_AVRO_READER_HPP_
2+
#define RFL_AVRO_READER_HPP_
3+
4+
#include <avro.h>
5+
6+
#include <cstddef>
7+
#include <exception>
8+
#include <string>
9+
#include <string_view>
10+
#include <type_traits>
11+
#include <vector>
12+
13+
#include "../Bytestring.hpp"
14+
#include "../Result.hpp"
15+
#include "../always_false.hpp"
16+
17+
namespace rfl ::avro {
18+
19+
/// Please refer to https://intel.github.io/tinyavro/current/index.html
20+
struct Reader {
21+
struct AVROInputArray {
22+
const avro_value_t* val_;
23+
};
24+
25+
struct AVROInputObject {
26+
const avro_value_t* val_;
27+
};
28+
29+
struct AVROInputVar {
30+
const avro_value_t* val_;
31+
};
32+
33+
using InputArrayType = AVROInputArray;
34+
using InputObjectType = AVROInputObject;
35+
using InputVarType = AVROInputVar;
36+
37+
template <class T>
38+
static constexpr bool has_custom_constructor =
39+
(requires(InputVarType var) { T::from_avro_obj(var); });
40+
41+
rfl::Result<InputVarType> get_field_from_array(
42+
const size_t _idx, const InputArrayType& _arr) const noexcept;
43+
44+
rfl::Result<InputVarType> get_field_from_object(
45+
const std::string& _name, const InputObjectType& _obj) const noexcept;
46+
47+
bool is_empty(const InputVarType& _var) const noexcept;
48+
49+
template <class T>
50+
rfl::Result<T> to_basic_type(const InputVarType& _var) const noexcept {
51+
const auto type = avro_value_get_type(_var.val_);
52+
if constexpr (std::is_same<std::remove_cvref_t<T>, std::string>()) {
53+
if (type != AVRO_STRING) {
54+
return Error("Could not cast to string.");
55+
}
56+
const char* c_str = nullptr;
57+
size_t size = 0;
58+
avro_value_get_string(_var.val_, &c_str, &size);
59+
return std::string(c_str, size);
60+
/*} else if constexpr (std::is_same<std::remove_cvref_t<T>,
61+
rfl::Bytestring>()) {
62+
if (!avro_value_is_byte_string(&_var.val_)) {
63+
return Error("Could not cast to bytestring.");
64+
}
65+
rfl::Bytestring bstr;
66+
const auto err = get_bytestring(&_var.val_, &bstr);
67+
if (err != AvroNoError) {
68+
return Error(avro_error_string(err));
69+
}
70+
return bstr;*/
71+
} else if constexpr (std::is_same<std::remove_cvref_t<T>, bool>()) {
72+
if (type != AVRO_BOOLEAN) {
73+
return rfl::Error("Could not cast to boolean.");
74+
}
75+
int result = 0;
76+
avro_value_get_boolean(_var.val_, &result);
77+
return (result != 0);
78+
} else if constexpr (std::is_floating_point<std::remove_cvref_t<T>>() ||
79+
std::is_integral<std::remove_cvref_t<T>>()) {
80+
if (type == AVRO_DOUBLE) {
81+
double result = 0.0;
82+
const auto err = avro_value_get_double(_var.val_, &result);
83+
return static_cast<T>(result);
84+
} else if (type == AVRO_INT32) {
85+
int32_t result = 0;
86+
avro_value_get_int(_var.val_, &result);
87+
return static_cast<T>(result);
88+
} else if (type == AVRO_INT64) {
89+
int64_t result = 0;
90+
avro_value_get_int64(_var.val_, &result);
91+
return static_cast<T>(result);
92+
} else if (type == AVRO_FLOAT) {
93+
double result = 0.0;
94+
const auto err = avro_value_get_float(_var.val_, &result);
95+
return static_cast<T>(result);
96+
}
97+
return rfl::Error(
98+
"Could not cast to numeric value. The type must be integral, float "
99+
"or double.");
100+
101+
} else {
102+
static_assert(rfl::always_false_v<T>, "Unsupported type.");
103+
}
104+
}
105+
106+
rfl::Result<InputArrayType> to_array(const InputVarType& _var) const noexcept;
107+
108+
rfl::Result<InputObjectType> to_object(
109+
const InputVarType& _var) const noexcept;
110+
111+
template <class ArrayReader>
112+
std::optional<Error> read_array(const ArrayReader& _array_reader,
113+
const InputArrayType& _arr) const noexcept {
114+
size_t size = 0;
115+
avro_value_get_size(_arr.val_, &size);
116+
for (size_t ix = 0; ix < size; ++ix) {
117+
avro_value_t element;
118+
avro_value_get_by_index(_arr.val_, ix, &element, nullptr);
119+
const auto err = _array_reader.read(InputVarType{&element});
120+
if (err) {
121+
return err;
122+
}
123+
}
124+
return std::nullopt;
125+
}
126+
127+
template <class ObjectReader>
128+
std::optional<Error> read_object(const ObjectReader& _object_reader,
129+
const InputObjectType& _obj) const noexcept {
130+
size_t size = 0;
131+
avro_value_get_size(_arr.val_, &size);
132+
for (size_t ix = 0; ix < size; ++ix) {
133+
avro_value_t element;
134+
const char* key = nullptr;
135+
avro_value_get_by_index(_obj.val_, ix, &element, &key);
136+
_object_reader.read(std::string_view(key), InputVarType{&element});
137+
}
138+
return std::nullopt;
139+
}
140+
141+
template <class T>
142+
rfl::Result<T> use_custom_constructor(
143+
const InputVarType& _var) const noexcept {
144+
try {
145+
return T::from_avro_obj(_var);
146+
} catch (std::exception& e) {
147+
return rfl::Error(e.what());
148+
}
149+
}
150+
};
151+
152+
} // namespace rfl::avro
153+
154+
#endif

‎include/rfl/avro/Writer.hpp‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#ifndef RFL_AVRO_WRITER_HPP_
2+
#define RFL_AVRO_WRITER_HPP_
3+
4+
#include <avro.h>
5+
6+
#include <bit>
7+
#include <exception>
8+
#include <map>
9+
#include <sstream>
10+
#include <stdexcept>
11+
#include <string>
12+
#include <string_view>
13+
#include <type_traits>
14+
#include <variant>
15+
#include <vector>
16+
17+
#include "../Box.hpp"
18+
#include "../Bytestring.hpp"
19+
#include "../Ref.hpp"
20+
#include "../Result.hpp"
21+
#include "../always_false.hpp"
22+
23+
namespace rfl {
24+
namespace avro {
25+
26+
class Writer {
27+
public:
28+
struct AVROOutputArray {
29+
AvroEncoder* encoder_;
30+
AvroEncoder* parent_;
31+
};
32+
33+
struct AVROOutputObject {
34+
AvroEncoder* encoder_;
35+
AvroEncoder* parent_;
36+
};
37+
38+
struct AVROOutputVar {};
39+
40+
using OutputArrayType = AVROOutputArray;
41+
using OutputObjectType = AVROOutputObject;
42+
using OutputVarType = AVROOutputVar;
43+
44+
Writer(AvroEncoder* _encoder);
45+
46+
~Writer();
47+
48+
OutputArrayType array_as_root(const size_t _size) const noexcept;
49+
50+
OutputObjectType object_as_root(const size_t _size) const noexcept;
51+
52+
OutputVarType null_as_root() const noexcept;
53+
54+
template <class T>
55+
OutputVarType value_as_root(const T& _var) const noexcept {
56+
return new_value(_var, encoder_);
57+
}
58+
59+
OutputArrayType add_array_to_array(const size_t _size,
60+
OutputArrayType* _parent) const noexcept;
61+
62+
OutputArrayType add_array_to_object(const std::string_view& _name,
63+
const size_t _size,
64+
OutputObjectType* _parent) const noexcept;
65+
66+
OutputObjectType add_object_to_array(const size_t _size,
67+
OutputArrayType* _parent) const noexcept;
68+
69+
OutputObjectType add_object_to_object(
70+
const std::string_view& _name, const size_t _size,
71+
OutputObjectType* _parent) const noexcept;
72+
73+
template <class T>
74+
OutputVarType add_value_to_array(const T& _var,
75+
OutputArrayType* _parent) const noexcept {
76+
return new_value(_var, _parent->encoder_);
77+
}
78+
79+
template <class T>
80+
OutputVarType add_value_to_object(const std::string_view& _name,
81+
const T& _var,
82+
OutputObjectType* _parent) const noexcept {
83+
avro_encode_text_string(_parent->encoder_, _name.data(), _name.size());
84+
return new_value(_var, _parent->encoder_);
85+
}
86+
87+
OutputVarType add_null_to_array(OutputArrayType* _parent) const noexcept;
88+
89+
OutputVarType add_null_to_object(const std::string_view& _name,
90+
OutputObjectType* _parent) const noexcept;
91+
92+
void end_array(OutputArrayType* _arr) const noexcept;
93+
94+
void end_object(OutputObjectType* _obj) const noexcept;
95+
96+
private:
97+
OutputArrayType new_array(const size_t _size,
98+
AvroEncoder* _parent) const noexcept;
99+
100+
OutputObjectType new_object(const size_t _size,
101+
AvroEncoder* _parent) const noexcept;
102+
103+
template <class T>
104+
OutputVarType new_value(const T& _var, AvroEncoder* _parent) const noexcept {
105+
if constexpr (std::is_same<std::remove_cvref_t<T>, std::string>()) {
106+
avro_encode_text_string(_parent, _var.c_str(), _var.size());
107+
} else if constexpr (std::is_same<std::remove_cvref_t<T>,
108+
rfl::Bytestring>()) {
109+
avro_encode_byte_string(
110+
_parent, std::bit_cast<const uint8_t*>(_var.c_str()), _var.size());
111+
} else if constexpr (std::is_same<std::remove_cvref_t<T>, bool>()) {
112+
avro_encode_boolean(_parent, _var);
113+
} else if constexpr (std::is_floating_point<std::remove_cvref_t<T>>()) {
114+
avro_encode_double(_parent, static_cast<double>(_var));
115+
} else if constexpr (std::is_integral<std::remove_cvref_t<T>>()) {
116+
avro_encode_int(_parent, static_cast<std::int64_t>(_var));
117+
} else {
118+
static_assert(rfl::always_false_v<T>, "Unsupported type.");
119+
}
120+
return OutputVarType{};
121+
}
122+
123+
private:
124+
/// The underlying TinyAVRO encoder.
125+
AvroEncoder* const encoder_;
126+
127+
/// Contain all of the subobjects and subarrays.
128+
const rfl::Box<std::vector<rfl::Box<AvroEncoder>>> subencoders_;
129+
};
130+
131+
} // namespace avro
132+
} // namespace rfl
133+
134+
#endif

0 commit comments

Comments
 (0)