Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions be/src/olap/rowset/segment_v2/binary_dict_page.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -142,19 +142,22 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) {
}
}

OwnedSlice BinaryDictPageBuilder::finish() {
Status BinaryDictPageBuilder::finish(OwnedSlice* slice) {
if (VLOG_DEBUG_IS_ON && _encoding_type == DICT_ENCODING) {
VLOG_DEBUG << "dict page size:" << _dict_builder->size();
}

DCHECK(!_finished);
_finished = true;

OwnedSlice data_slice = _data_page_builder->finish();
OwnedSlice data_slice;
RETURN_IF_ERROR(_data_page_builder->finish(&data_slice));
// TODO(gaodayue) separate page header and content to avoid this copy
_buffer.append(data_slice.slice().data, data_slice.slice().size);
RETURN_IF_CATCH_EXCEPTION(
{ _buffer.append(data_slice.slice().data, data_slice.slice().size); });
encode_fixed32_le(&_buffer[0], _encoding_type);
return _buffer.build();
*slice = _buffer.build();
return Status::OK();
}

Status BinaryDictPageBuilder::reset() {
Expand Down Expand Up @@ -185,8 +188,7 @@ uint64_t BinaryDictPageBuilder::size() const {
}

Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page) {
*dictionary_page = _dict_builder->finish();
return Status::OK();
return _dict_builder->finish(dictionary_page);
}

Status BinaryDictPageBuilder::get_first_value(void* value) const {
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/binary_dict_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class BinaryDictPageBuilder : public PageBuilderHelper<BinaryDictPageBuilder> {

Status add(const uint8_t* vals, size_t* count) override;

OwnedSlice finish() override;
Status finish(OwnedSlice* slice) override;

Status reset() override;

Expand Down
25 changes: 14 additions & 11 deletions be/src/olap/rowset/segment_v2/binary_plain_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -93,19 +93,22 @@ class BinaryPlainPageBuilder : public PageBuilderHelper<BinaryPlainPageBuilder<T
return Status::OK();
}

OwnedSlice finish() override {
Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
// Set up trailer
for (uint32_t _offset : _offsets) {
put_fixed32_le(&_buffer, _offset);
}
put_fixed32_le(&_buffer, _offsets.size());
if (_offsets.size() > 0) {
_copy_value_at(0, &_first_value);
_copy_value_at(_offsets.size() - 1, &_last_value);
}
return _buffer.build();
RETURN_IF_CATCH_EXCEPTION({
// Set up trailer
for (uint32_t _offset : _offsets) {
put_fixed32_le(&_buffer, _offset);
}
put_fixed32_le(&_buffer, _offsets.size());
if (_offsets.size() > 0) {
_copy_value_at(0, &_first_value);
_copy_value_at(_offsets.size() - 1, &_last_value);
}
*slice = _buffer.build();
});
return Status::OK();
}

Status reset() override {
Expand Down
23 changes: 13 additions & 10 deletions be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,21 @@ Status BinaryPrefixPageBuilder::add(const uint8_t* vals, size_t* add_count) {
return Status::OK();
}

OwnedSlice BinaryPrefixPageBuilder::finish() {
Status BinaryPrefixPageBuilder::finish(OwnedSlice* slice) {
DCHECK(!_finished);
_finished = true;
put_fixed32_le(&_buffer, (uint32_t)_count);
uint8_t restart_point_internal = RESTART_POINT_INTERVAL;
_buffer.append(&restart_point_internal, 1);
auto restart_point_size = _restart_points_offset.size();
for (uint32_t i = 0; i < restart_point_size; ++i) {
put_fixed32_le(&_buffer, _restart_points_offset[i]);
}
put_fixed32_le(&_buffer, restart_point_size);
return _buffer.build();
RETURN_IF_CATCH_EXCEPTION({
put_fixed32_le(&_buffer, (uint32_t)_count);
uint8_t restart_point_internal = RESTART_POINT_INTERVAL;
_buffer.append(&restart_point_internal, 1);
auto restart_point_size = _restart_points_offset.size();
for (uint32_t i = 0; i < restart_point_size; ++i) {
put_fixed32_le(&_buffer, _restart_points_offset[i]);
}
put_fixed32_le(&_buffer, restart_point_size);
*slice = _buffer.build();
});
return Status::OK();
}

const uint8_t* BinaryPrefixPageDecoder::_decode_value_lengths(const uint8_t* ptr, uint32_t* shared,
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/binary_prefix_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class BinaryPrefixPageBuilder : public PageBuilderHelper<BinaryPrefixPageBuilder

Status add(const uint8_t* vals, size_t* add_count) override;

OwnedSlice finish() override;
Status finish(OwnedSlice* slice) override;

Status reset() override {
_restart_points_offset.clear();
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/rowset/segment_v2/bitshuffle_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,13 @@ class BitshufflePageBuilder : public PageBuilderHelper<BitshufflePageBuilder<Typ
return Status::OK();
}

OwnedSlice finish() override {
Status finish(OwnedSlice* slice) override {
if (_count > 0) {
_first_value = cell(0);
_last_value = cell(_count - 1);
}
return _finish(SIZE_OF_TYPE);
RETURN_IF_CATCH_EXCEPTION({ *slice = _finish(SIZE_OF_TYPE); });
return Status::OK();
}

Status reset() override {
Expand Down
10 changes: 6 additions & 4 deletions be/src/olap/rowset/segment_v2/column_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,10 @@ class NullBitmapBuilder {
// Returns whether the building nullmap contains nullptr
bool has_null() const { return _has_null; }

OwnedSlice finish() {
Status finish(OwnedSlice* slice) {
_rle_encoder.Flush();
return _bitmap_buf.build();
RETURN_IF_CATCH_EXCEPTION({ *slice = _bitmap_buf.build(); });
return Status::OK();
}

void reset() {
Expand Down Expand Up @@ -676,14 +677,15 @@ Status ScalarColumnWriter::finish_current_page() {

// build data page body : encoded values + [nullmap]
std::vector<Slice> body;
OwnedSlice encoded_values = _page_builder->finish();
OwnedSlice encoded_values;
RETURN_IF_ERROR(_page_builder->finish(&encoded_values));
RETURN_IF_ERROR(_page_builder->reset());
body.push_back(encoded_values.slice());

OwnedSlice nullmap;
if (_null_bitmap_builder != nullptr) {
if (is_nullable() && _null_bitmap_builder->has_null()) {
nullmap = _null_bitmap_builder->finish();
RETURN_IF_ERROR(_null_bitmap_builder->finish(&nullmap));
body.push_back(nullmap.slice());
}
_null_bitmap_builder->reset();
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/rowset/segment_v2/frame_of_reference_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,12 @@ class FrameOfReferencePageBuilder : public PageBuilderHelper<FrameOfReferencePag
return Status::OK();
}

OwnedSlice finish() override {
Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
_encoder->flush();
return _buf.build();
RETURN_IF_CATCH_EXCEPTION({ *slice = _buf.build(); });
return Status::OK();
}

Status reset() override {
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@ Status IndexedColumnWriter::_finish_current_data_page(size_t& num_val) {
ordinal_t first_ordinal = _num_values - num_values_in_page;

// IndexedColumn doesn't have NULLs, thus data page body only contains encoded values
OwnedSlice page_body = _data_page_builder->finish();
OwnedSlice page_body;
RETURN_IF_ERROR(_data_page_builder->finish(&page_body));
RETURN_IF_ERROR(_data_page_builder->reset());

PageFooterPB footer;
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/rowset/segment_v2/page_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ class PageBuilder {

// Finish building the current page, return the encoded data.
// This api should be followed by reset() before reusing the builder
virtual OwnedSlice finish() = 0;
// It will return error status when memory allocated failed during finish
virtual Status finish(OwnedSlice* owned_slice) = 0;

// Get the dictionary page for dictionary encoding mode column.
virtual Status get_dictionary_page(OwnedSlice* dictionary_page) {
Expand Down
18 changes: 11 additions & 7 deletions be/src/olap/rowset/segment_v2/plain_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,18 @@ class PlainPageBuilder : public PageBuilderHelper<PlainPageBuilder<Type> > {
return Status::OK();
}

OwnedSlice finish() override {
Status finish(OwnedSlice* slice) override {
encode_fixed32_le((uint8_t*)&_buffer[0], _count);
if (_count > 0) {
_first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE], SIZE_OF_TYPE);
_last_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE + (_count - 1) * SIZE_OF_TYPE],
SIZE_OF_TYPE);
}
return _buffer.build();
RETURN_IF_CATCH_EXCEPTION({
if (_count > 0) {
_first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE], SIZE_OF_TYPE);
_last_value.assign_copy(
&_buffer[PLAIN_PAGE_HEADER_SIZE + (_count - 1) * SIZE_OF_TYPE],
SIZE_OF_TYPE);
}
*slice = _buffer.build();
});
return Status::OK();
}

Status reset() override {
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/rowset/segment_v2/rle_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,15 @@ class RlePageBuilder : public PageBuilderHelper<RlePageBuilder<Type> > {
return Status::OK();
}

OwnedSlice finish() override {
Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
// here should Flush first and then encode the count header
// or it will lead to a bug if the header is less than 8 byte and the data is small
_rle_encoder->Flush();
encode_fixed32_le(&_buf[0], _count);
return _buf.build();
*slice = _buf.build();
return Status::OK();
}

Status reset() override {
Expand Down