feat(puffin): add basic data structures and constants

src/iceberg/CMakeLists.txt

@@ -62,6 +62,10 @@ set(ICEBERG_SOURCES
     partition_field.cc
     partition_spec.cc
     partition_summary.cc
+    puffin/blob.cc
+    puffin/blob_metadata.cc
+    puffin/file_metadata.cc
+    puffin/puffin_compression_codec.cc
     row/arrow_array_wrapper.cc
     row/manifest_wrapper.cc
     row/partition_values.cc
@@ -166,6 +170,7 @@ add_subdirectory(catalog)
 add_subdirectory(data)
 add_subdirectory(expression)
 add_subdirectory(manifest)
+add_subdirectory(puffin)
 add_subdirectory(row)
 add_subdirectory(update)
 add_subdirectory(util)

src/iceberg/meson.build

@@ -80,6 +80,10 @@ iceberg_sources = files(
     'partition_field.cc',
     'partition_spec.cc',
     'partition_summary.cc',
+    'puffin/blob.cc',
+    'puffin/blob_metadata.cc',
+    'puffin/file_metadata.cc',
+    'puffin/puffin_compression_codec.cc',
     'row/arrow_array_wrapper.cc',
     'row/manifest_wrapper.cc',
     'row/partition_values.cc',
@@ -221,6 +225,7 @@ install_headers(
 subdir('catalog')
 subdir('expression')
 subdir('manifest')
+subdir('puffin')
 subdir('row')
 subdir('update')
 subdir('util')

src/iceberg/puffin/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+iceberg_install_all_headers(iceberg/puffin)

src/iceberg/puffin/blob.cc

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/blob.h"
+
+#include <format>
+
+#include "iceberg/util/formatter_internal.h"
+
+namespace iceberg::puffin {
+
+std::string ToString(const Blob& blob) {
+  std::string repr = "Blob[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},", blob.type,
+                 blob.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob.snapshot_id, blob.sequence_number);
+  std::format_to(std::back_inserter(repr), "dataSize={}", blob.data.size());
+  if (blob.requested_compression.has_value()) {
+    std::format_to(std::back_inserter(repr), ",requestedCompression={}",
+                   iceberg::puffin::ToString(*blob.requested_compression));
+  }
+  if (!blob.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/blob.h

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/blob.h
+/// Blob data structure for Puffin files.
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/puffin/puffin_compression_codec.h"
+
+namespace iceberg::puffin {
+
+/// \brief A blob to be written to a Puffin file.
+///
+/// This represents the uncompressed blob data along with its metadata.
+/// The actual compression is handled during writing.
+struct ICEBERG_EXPORT Blob {
+  /// Type of the blob. See StandardBlobTypes for known types.
+  std::string type;
+  /// List of field IDs the blob was computed for.
+  /// The order of items is used to compute sketches stored in the blob.
+  std::vector<int32_t> input_fields;
+  /// ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  /// The uncompressed blob data.
+  std::vector<uint8_t> data;
+  /// Requested compression codec. If not set, the writer's default will be used.
+  std::optional<PuffinCompressionCodec> requested_compression;
+  /// Additional properties of the blob.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two Blobs for equality.
+  friend bool operator==(const Blob& lhs, const Blob& rhs) = default;
+};
+
+/// \brief Returns a string representation of a Blob.
+ICEBERG_EXPORT std::string ToString(const Blob& blob);
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/blob_metadata.cc

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/blob_metadata.h"
+
+#include <format>
+
+#include "iceberg/util/formatter_internal.h"
+
+namespace iceberg::puffin {
+
+std::string ToString(const BlobMetadata& blob_metadata) {
+  std::string repr = "BlobMetadata[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},",
+                 blob_metadata.type, blob_metadata.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob_metadata.snapshot_id, blob_metadata.sequence_number);
+  std::format_to(std::back_inserter(repr), "offset={},length={}", blob_metadata.offset,
+                 blob_metadata.length);
+  if (blob_metadata.compression_codec.has_value()) {
+    std::format_to(std::back_inserter(repr), ",compressionCodec='{}'",
+                   *blob_metadata.compression_codec);
+  }
+  if (!blob_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/blob_metadata.h

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/blob_metadata.h
+/// Blob metadata structure for Puffin files.
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+
+namespace iceberg::puffin {
+
+/// \brief Metadata about a blob stored in a Puffin file.
+///
+/// This represents the metadata stored in the Puffin file footer,
+/// including the blob's location within the file.
+struct ICEBERG_EXPORT BlobMetadata {
+  /// Type of the blob. See StandardBlobTypes for known types.
+  std::string type;
+  /// List of field IDs the blob was computed for.
+  std::vector<int32_t> input_fields;
+  /// ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  /// Offset in the file where the blob data starts.
+  int64_t offset;
+  /// Length of the blob data in the file (after compression, if compressed).
+  int64_t length;
+  /// Compression codec name, or std::nullopt if uncompressed.
+  std::optional<std::string> compression_codec;
+  /// Additional properties of the blob.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two BlobMetadatas for equality.
+  friend bool operator==(const BlobMetadata& lhs, const BlobMetadata& rhs) = default;
+};
+
+/// \brief Returns a string representation of a BlobMetadata.
+ICEBERG_EXPORT std::string ToString(const BlobMetadata& blob_metadata);
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/file_metadata.cc

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/file_metadata.h"
+
+#include <format>
+
+#include "iceberg/util/formatter_internal.h"
+
+namespace iceberg::puffin {
+
+std::string ToString(const FileMetadata& file_metadata) {
+  std::string repr = "FileMetadata[";
+  std::format_to(std::back_inserter(repr), "blobs=[");
+  for (size_t i = 0; i < file_metadata.blobs.size(); ++i) {
+    if (i > 0) {
+      std::format_to(std::back_inserter(repr), ",");
+    }
+    std::format_to(std::back_inserter(repr), "{}", ToString(file_metadata.blobs[i]));
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  if (!file_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", file_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/file_metadata.h

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/file_metadata.h
+/// File metadata structure for Puffin files.
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/puffin/blob_metadata.h"
+
+namespace iceberg::puffin {
+
+/// \brief Metadata about a Puffin file.
+///
+/// This represents the metadata stored in the Puffin file footer,
+/// including information about all blobs in the file.
+struct ICEBERG_EXPORT FileMetadata {
+  /// List of blob metadata for all blobs in the file.
+  std::vector<BlobMetadata> blobs;
+  /// File-level properties.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two FileMetadatas for equality.
+  friend bool operator==(const FileMetadata& lhs, const FileMetadata& rhs) = default;
+};
+
+/// \brief Returns a string representation of a FileMetadata.
+ICEBERG_EXPORT std::string ToString(const FileMetadata& file_metadata);
+
+}  // namespace iceberg::puffin