diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index 041e84837..ff98ae6a7 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -26,7 +26,7 @@ mkdir ${build_dir} pushd ${build_dir} is_windows() { - [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" || "${OSTYPE}" == "cygwin" ]] } CMAKE_ARGS=( diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index cf332c21c..5d69dd074 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -28,7 +28,7 @@ mkdir ${build_dir} pushd ${build_dir} is_windows() { - [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" || "${OSTYPE}" == "cygwin" ]] } CMAKE_ARGS=( diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 837ed7d7c..370d916a1 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -22,7 +22,7 @@ project(example) set(CMAKE_CXX_STANDARD 23) -find_package(iceberg CONFIG REQUIRED) +find_package(iceberg CONFIG REQUIRED COMPONENTS bundle rest) add_executable(demo_example demo_example.cc) diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 603c35343..c040d0160 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -20,14 +20,7 @@ set(ICEBERG_INCLUDES "$" set(ICEBERG_SOURCES arrow_c_data_guard_internal.cc catalog/memory/in_memory_catalog.cc - data/data_writer.cc - data/delete_loader.cc - data/equality_delete_writer.cc - data/position_delete_writer.cc - data/writer.cc delete_file_index.cc - deletes/roaring_position_bitmap.cc - deletes/position_delete_index.cc expression/aggregate.cc expression/binder.cc expression/evaluator.cc @@ -66,9 +59,6 @@ set(ICEBERG_SOURCES partition_field.cc partition_spec.cc partition_summary.cc - puffin/file_metadata.cc - puffin/puffin_format.cc - puffin/json_serde.cc row/arrow_array_wrapper.cc row/manifest_wrapper.cc row/partition_values.cc @@ -134,24 +124,22 @@ list(APPEND ICEBERG_STATIC_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow_static nlohmann_json::nlohmann_json - roaring::roaring ZLIB::ZLIB) list(APPEND ICEBERG_SHARED_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow_shared nlohmann_json::nlohmann_json - roaring::roaring ZLIB::ZLIB) list(APPEND ICEBERG_STATIC_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_static,$,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::roaring,roaring::roaring>") +) list(APPEND ICEBERG_SHARED_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_shared,$,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::roaring,roaring::roaring>") +) add_iceberg_lib(iceberg SOURCES @@ -169,6 +157,60 @@ add_iceberg_lib(iceberg OUTPUTS ICEBERG_LIBRARIES) +set(ICEBERG_DATA_SOURCES + data/data_writer.cc + data/delete_loader.cc + data/equality_delete_writer.cc + data/position_delete_writer.cc + data/writer.cc + deletes/position_delete_index.cc + deletes/roaring_position_bitmap.cc + puffin/file_metadata.cc + puffin/json_serde.cc + puffin/puffin_format.cc) + +set(ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS) +set(ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS) +set(ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS) +set(ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS) + +list(APPEND ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + roaring::roaring) +list(APPEND ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + roaring::roaring) +list(APPEND + ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS + "$,iceberg::iceberg_static,iceberg::iceberg_shared>" + "$,iceberg::roaring,roaring::roaring>") +list(APPEND + ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS + "$,iceberg::iceberg_shared,iceberg::iceberg_static>" + "$,iceberg::roaring,roaring::roaring>") + +add_iceberg_lib(iceberg_data + SOURCES + ${ICEBERG_DATA_SOURCES} + EXTRA_INCLUDES + ${ICEBERG_INCLUDES} + SHARED_LINK_LIBS + ${ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS} + STATIC_LINK_LIBS + ${ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS} + STATIC_INSTALL_INTERFACE_LIBS + ${ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_INSTALL_INTERFACE_LIBS + ${ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS}) + +if(TARGET iceberg_data_shared) + target_compile_definitions(iceberg_data_shared PRIVATE ICEBERG_EXPORTING) +endif() + +if(TARGET iceberg_data_static) + target_compile_definitions(iceberg_data_static PRIVATE ICEBERG_STATIC) +endif() + iceberg_install_all_headers(iceberg) add_subdirectory(catalog) @@ -210,14 +252,14 @@ if(ICEBERG_BUILD_BUNDLE) list(APPEND ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS - "$,iceberg_static,iceberg_shared>" + "$,iceberg_data_static,iceberg_data_shared>" "$,Arrow::arrow_static,Arrow::arrow_shared>" "$,Parquet::parquet_static,Parquet::parquet_shared>" "$,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>" ) list(APPEND ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS - "$,iceberg_shared,iceberg_static>" + "$,iceberg_data_shared,iceberg_data_static>" "$,Arrow::arrow_shared,Arrow::arrow_static>" "$,Parquet::parquet_shared,Parquet::parquet_static>" "$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>" @@ -225,14 +267,14 @@ if(ICEBERG_BUILD_BUNDLE) list(APPEND ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_static,iceberg::iceberg_shared>" + "$,iceberg::iceberg_data_static,iceberg::iceberg_data_shared>" "$,iceberg::arrow_static,$,Arrow::arrow_static,Arrow::arrow_shared>>" "$,iceberg::parquet_static,$,Parquet::parquet_static,Parquet::parquet_shared>>" "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>>" ) list(APPEND ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_shared,iceberg::iceberg_static>" + "$,iceberg::iceberg_data_shared,iceberg::iceberg_data_static>" "$,iceberg::arrow_static,$,Arrow::arrow_shared,Arrow::arrow_static>>" "$,iceberg::parquet_static,$,Parquet::parquet_shared,Parquet::parquet_static>>" "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>" diff --git a/src/iceberg/arrow_c_data_guard_internal.h b/src/iceberg/arrow_c_data_guard_internal.h index 8bce14e57..04603b0fe 100644 --- a/src/iceberg/arrow_c_data_guard_internal.h +++ b/src/iceberg/arrow_c_data_guard_internal.h @@ -22,10 +22,11 @@ #include #include "iceberg/arrow_c_data.h" +#include "iceberg/iceberg_export.h" namespace iceberg::internal { -class ArrowArrayGuard { +class ICEBERG_EXPORT ArrowArrayGuard { public: explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {} ~ArrowArrayGuard(); @@ -34,7 +35,7 @@ class ArrowArrayGuard { ArrowArray* array_; }; -class ArrowSchemaGuard { +class ICEBERG_EXPORT ArrowSchemaGuard { public: explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {} ~ArrowSchemaGuard(); @@ -43,7 +44,7 @@ class ArrowSchemaGuard { ArrowSchema* schema_; }; -class ArrowArrayViewGuard { +class ICEBERG_EXPORT ArrowArrayViewGuard { public: explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {} ~ArrowArrayViewGuard(); @@ -52,7 +53,7 @@ class ArrowArrayViewGuard { ArrowArrayView* view_; }; -class ArrowArrayBufferGuard { +class ICEBERG_EXPORT ArrowArrayBufferGuard { public: explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {} ~ArrowArrayBufferGuard(); diff --git a/src/iceberg/data/meson.build b/src/iceberg/data/meson.build new file mode 100644 index 000000000..0f68deccf --- /dev/null +++ b/src/iceberg/data/meson.build @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +install_headers( + [ + 'data_writer.h', + 'delete_loader.h', + 'equality_delete_writer.h', + 'position_delete_writer.h', + 'writer.h', + ], + subdir: 'iceberg/data', +) diff --git a/src/iceberg/file_writer.h b/src/iceberg/file_writer.h index 0b1af7f93..f3352d8fd 100644 --- a/src/iceberg/file_writer.h +++ b/src/iceberg/file_writer.h @@ -36,7 +36,7 @@ namespace iceberg { -class WriterProperties : public ConfigBase { +class ICEBERG_EXPORT WriterProperties : public ConfigBase { public: template using Entry = const ConfigBase::Entry; diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in index 787fadcc6..7a46cdd1d 100644 --- a/src/iceberg/iceberg-config.cmake.in +++ b/src/iceberg/iceberg-config.cmake.in @@ -24,6 +24,8 @@ # # iceberg::iceberg_shared # iceberg::iceberg_static +# iceberg::iceberg_data_shared +# iceberg::iceberg_data_static # iceberg::iceberg_bundle_shared # iceberg::iceberg_bundle_static # iceberg::iceberg_rest_shared diff --git a/src/iceberg/iceberg_export.h b/src/iceberg/iceberg_export.h index 8d7a35f27..2463fbeaf 100644 --- a/src/iceberg/iceberg_export.h +++ b/src/iceberg/iceberg_export.h @@ -53,5 +53,5 @@ # define ICEBERG_TEMPLATE_EXPORT # define ICEBERG_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT -# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_EXPORT +# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_CLASS_EXPORT #endif diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 5e68def98..72ea60e4e 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -42,14 +42,7 @@ iceberg_include_dir = include_directories('..') iceberg_sources = files( 'arrow_c_data_guard_internal.cc', 'catalog/memory/in_memory_catalog.cc', - 'data/data_writer.cc', - 'data/delete_loader.cc', - 'data/equality_delete_writer.cc', - 'data/position_delete_writer.cc', - 'data/writer.cc', 'delete_file_index.cc', - 'deletes/position_delete_index.cc', - 'deletes/roaring_position_bitmap.cc', 'expression/aggregate.cc', 'expression/binder.cc', 'expression/evaluator.cc', @@ -88,9 +81,6 @@ iceberg_sources = files( 'partition_field.cc', 'partition_spec.cc', 'partition_summary.cc', - 'puffin/file_metadata.cc', - 'puffin/json_serde.cc', - 'puffin/puffin_format.cc', 'row/arrow_array_wrapper.cc', 'row/manifest_wrapper.cc', 'row/partition_values.cc', @@ -148,6 +138,19 @@ iceberg_sources = files( 'util/uuid.cc', ) +iceberg_data_sources = files( + 'data/data_writer.cc', + 'data/delete_loader.cc', + 'data/equality_delete_writer.cc', + 'data/position_delete_writer.cc', + 'data/writer.cc', + 'deletes/position_delete_index.cc', + 'deletes/roaring_position_bitmap.cc', + 'puffin/file_metadata.cc', + 'puffin/json_serde.cc', + 'puffin/puffin_format.cc', +) + # CRoaring does not export symbols, so on Windows it must # be used as a static lib croaring_needs_static = ( @@ -159,7 +162,7 @@ nanoarrow_dep = dependency('nanoarrow') nlohmann_json_dep = dependency('nlohmann_json') zlib_dep = dependency('zlib') -iceberg_deps = [croaring_dep, nanoarrow_dep, nlohmann_json_dep, zlib_dep] +iceberg_deps = [nanoarrow_dep, nlohmann_json_dep, zlib_dep] iceberg_lib = library( 'iceberg', @@ -184,8 +187,35 @@ iceberg_dep = declare_dependency( compile_args: iceberg_interface_args, ) meson.override_dependency('iceberg', iceberg_dep) + +iceberg_data_deps = [iceberg_dep, croaring_dep] +iceberg_data_lib = library( + 'iceberg_data', + sources: iceberg_data_sources, + dependencies: iceberg_data_deps, + include_directories: iceberg_include_dir, + install: true, + gnu_symbol_visibility: 'inlineshidden', + cpp_shared_args: ['-DICEBERG_EXPORTING'], + cpp_static_args: ['-DICEBERG_STATIC'], +) + +iceberg_data_interface_args = [] +if get_option('default_library') != 'shared' + iceberg_data_interface_args += ['-DICEBERG_STATIC'] +endif + +iceberg_data_dep = declare_dependency( + link_with: iceberg_data_lib, + dependencies: iceberg_data_deps, + include_directories: iceberg_include_dir, + compile_args: iceberg_data_interface_args, +) +meson.override_dependency('iceberg-data', iceberg_data_dep) + pkg = import('pkgconfig') pkg.generate(iceberg_lib) +pkg.generate(iceberg_data_lib) install_headers( [ @@ -233,6 +263,7 @@ install_headers( ) subdir('catalog') +subdir('data') subdir('deletes') subdir('expression') subdir('manifest') diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index b0f03b527..64d2d4758 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -31,7 +31,7 @@ set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources") configure_file("test_config.h.in" "test_config.h") function(add_iceberg_test test_name) - set(options USE_BUNDLE) + set(options USE_BUNDLE USE_DATA) set(oneValueArgs) set(multiValueArgs SOURCES) cmake_parse_arguments(ARG @@ -47,6 +47,10 @@ function(add_iceberg_test test_name) if(ARG_USE_BUNDLE) target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gmock_main) + elseif(ARG_USE_DATA) + target_link_libraries(${test_name} + PRIVATE "$,iceberg_data_static,iceberg_data_shared>" + GTest::gmock_main) else() target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gmock_main) endif() @@ -109,6 +113,7 @@ add_iceberg_test(json_serde_test schema_json_test.cc) add_iceberg_test(util_test + USE_DATA SOURCES bucket_util_test.cc config_test.cc @@ -128,9 +133,13 @@ add_iceberg_test(util_test uuid_test.cc visit_type_test.cc) -add_iceberg_test(roaring_test SOURCES roaring_test.cc) +add_iceberg_test(roaring_test USE_DATA SOURCES roaring_test.cc) -add_iceberg_test(puffin_test SOURCES puffin_format_test.cc puffin_json_test.cc) +add_iceberg_test(puffin_test + USE_DATA + SOURCES + puffin_format_test.cc + puffin_json_test.cc) if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(avro_test diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index ed9a5866a..b7385a912 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -101,10 +101,12 @@ iceberg_tests = { 'uuid_test.cc', 'visit_type_test.cc', ), + 'use_data': true, }, - 'roaring_test': {'sources': files('roaring_test.cc')}, + 'roaring_test': {'sources': files('roaring_test.cc'), 'use_data': true}, 'puffin_test': { 'sources': files('puffin_format_test.cc', 'puffin_json_test.cc'), + 'use_data': true, }, } @@ -140,13 +142,17 @@ if get_option('rest').enabled() endif foreach test_name, values : iceberg_tests + test_deps = [gmock_main_dep] + values.get('dependencies', []) + if values.get('use_data', false) + test_deps = [iceberg_data_dep] + test_deps + else + test_deps = [iceberg_dep] + test_deps + endif + exc = executable( test_name, sources: values['sources'], - dependencies: [iceberg_dep, gmock_main_dep] + values.get( - 'dependencies', - [], - ), + dependencies: test_deps, ) test(test_name, exc) endforeach