| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one or more | |
| * contributor license agreements. See the NOTICE file distributed with | |
| * this work for additional information regarding copyright ownership. | |
| * The ASF licenses this file to You under the Apache License, Version 2.0 | |
| * (the "License"); you may not use this file except in compliance with | |
| * the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| /* An independent C client: layouts and function signatures come from Spark's actual header. */ | |
| #include "spark_datasource.h" | |
| #include <dlfcn.h> | |
| #include <inttypes.h> | |
| #include <pthread.h> | |
| #include <stddef.h> | |
| #include <stdio.h> | |
| #include <stdlib.h> | |
| #include <string.h> | |
| #define CHECK(condition, message) do { \ | |
| if (!(condition)) { \ | |
| fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, (message)); \ | |
| exit(1); \ | |
| } \ | |
| } while (0) | |
| typedef struct Bytes { | |
| uint8_t* data; | |
| int64_t length; | |
| } Bytes; | |
| typedef struct ByteList { | |
| Bytes* values; | |
| size_t length; | |
| } ByteList; | |
| /* What a planning call returns: its partitions, and the state of its reader, set once. */ | |
| typedef struct Plan { | |
| ByteList partitions; | |
| ByteList states; | |
| } Plan; | |
| static const SparkDsLibrary* library; | |
| static const SparkDsTaskContext* expected_task; | |
| static bool interrupted; | |
| static int interruption_checks; | |
| static void check_status(int status) { | |
| if (status != 0) { | |
| const char* error = library->last_error(); | |
| fprintf(stderr, "Native status %d: %s\n", status, error == NULL ? "no message" : error); | |
| exit(1); | |
| } | |
| } | |
| static void log_message(int32_t level, const char* message) { | |
| fprintf(stderr, "Native log %d: %s\n", level, message); | |
| } | |
| static bool is_log_enabled(int32_t level) { | |
| return level <= SPARK_DS_LOG_INFO; | |
| } | |
| static bool is_interrupted(const SparkDsTaskContext* task) { | |
| CHECK(task == expected_task, "Cancellation callback must receive the original task pointer"); | |
| CHECK(task->stage_id == 71 && task->stage_attempt_number == 2, "Unexpected stage identity"); | |
| CHECK(task->partition_id == 3 && task->attempt_number == 4, "Unexpected task identity"); | |
| CHECK(task->task_attempt_id == INT64_C(12345678901), "Task attempt ID was truncated"); | |
| interruption_checks++; | |
| return interrupted; | |
| } | |
| static bool unavailable_callback(const SparkDsTaskContext* task) { | |
| (void) task; | |
| CHECK(false, "A task callback outside its declared struct size must not be read"); | |
| return false; | |
| } | |
| static SparkDsTaskContext task_context(void) { | |
| SparkDsTaskContext task = { | |
| .size = sizeof(SparkDsTaskContext), | |
| .stage_id = 71, | |
| .stage_attempt_number = 2, | |
| .partition_id = 3, | |
| .attempt_number = 4, | |
| .task_attempt_id = INT64_C(12345678901), | |
| .is_interrupted = is_interrupted | |
| }; | |
| return task; | |
| } | |
| /* Sink callbacks copy opaque bytes during the call, as required by the ABI. */ | |
| static void append(ByteList* list, const uint8_t* data, int64_t length) { | |
| CHECK(length >= 0, "A sink received a negative length"); | |
| CHECK(length == 0 || data != NULL, "A nonempty sink value must have data"); | |
| Bytes* values = realloc(list->values, (list->length + 1) * sizeof(Bytes)); | |
| CHECK(values != NULL, "Cannot allocate sink values"); | |
| list->values = values; | |
| Bytes* value = &list->values[list->length++]; | |
| value->length = length; | |
| value->data = malloc(length == 0 ? 1 : (size_t) length); | |
| CHECK(value->data != NULL, "Cannot allocate sink data"); | |
| if (length != 0) { | |
| memcpy(value->data, data, (size_t) length); | |
| } | |
| } | |
| static void add_bytes(void* sink, const uint8_t* data, int64_t length) { | |
| append(sink, data, length); | |
| } | |
| /* The sinks of a planning call share its sink argument, a Plan. */ | |
| static void add_partition(void* sink, const uint8_t* data, int64_t length) { | |
| append(&((Plan*) sink)->partitions, data, length); | |
| } | |
| static void set_state(void* sink, const uint8_t* data, int64_t length) { | |
| append(&((Plan*) sink)->states, data, length); | |
| } | |
| static void free_list(ByteList* list) { | |
| for (size_t i = 0; i < list->length; i++) { | |
| free(list->values[i].data); | |
| } | |
| free(list->values); | |
| list->values = NULL; | |
| list->length = 0; | |
| } | |
| static void free_plan(Plan* plan) { | |
| free_list(&plan->partitions); | |
| free_list(&plan->states); | |
| } | |
| static void release_field(struct ArrowSchema* schema) { | |
| schema->release = NULL; | |
| } | |
| static void release_schema(struct ArrowSchema* schema) { | |
| for (int64_t i = 0; i < schema->n_children; i++) { | |
| free(schema->children[i]); | |
| } | |
| free(schema->children); | |
| schema->release = NULL; | |
| } | |
| /* The schema to read that Spark passes for these columns of the Range source. */ | |
| static struct ArrowSchema read_schema(const char* const* columns, int64_t count) { | |
| struct ArrowSchema schema = { | |
| .format = "+s", | |
| .name = "", | |
| .n_children = count, | |
| .children = calloc(count == 0 ? 1 : (size_t) count, sizeof(struct ArrowSchema*)), | |
| .release = release_schema | |
| }; | |
| CHECK(schema.children != NULL, "Cannot allocate a schema"); | |
| for (int64_t i = 0; i < count; i++) { | |
| struct ArrowSchema* field = calloc(1, sizeof(struct ArrowSchema)); | |
| CHECK(field != NULL, "Cannot allocate a field"); | |
| bool id = strcmp(columns[i], "id") == 0; | |
| field->format = id ? "l" : "u"; | |
| field->name = id ? "id" : "label"; | |
| field->release = release_field; | |
| schema.children[i] = field; | |
| } | |
| return schema; | |
| } | |
| /* Releases an input schema unless the library moved it, as Spark does after a call. */ | |
| static void release_input(struct ArrowSchema* schema) { | |
| if (schema->release != NULL) { | |
| schema->release(schema); | |
| } | |
| } | |
| static void* create_source(const char* const* keys, const char* const* values, int64_t count) { | |
| SparkDsCreateInfo info = { | |
| .size = sizeof(SparkDsCreateInfo), | |
| .name = "rust_sdk_range", | |
| .option_keys = keys, | |
| .option_values = values, | |
| .num_options = count | |
| }; | |
| uint64_t capabilities = 0; | |
| void* source = NULL; | |
| check_status(library->create(&info, &capabilities, &source)); | |
| CHECK(source != NULL, "create returned a null source"); | |
| CHECK(capabilities == | |
| (SPARK_DS_BATCH_READ | SPARK_DS_MICRO_BATCH_READ | SPARK_DS_ADMISSION_CONTROL), | |
| "The Range source must declare batch and micro-batch reads, with admission control"); | |
| return source; | |
| } | |
| static void check_capability_discovery( | |
| const char* const* keys, const char* const* values, int64_t count) { | |
| /* Analysis may create a source only to inspect its capabilities, then immediately close it. */ | |
| void* source = create_source(keys, values, count); | |
| library->close(source); | |
| } | |
| static void* create_planner( | |
| const char* const* keys, | |
| const char* const* values, | |
| int64_t option_count) { | |
| void* source = create_source(keys, values, option_count); | |
| struct ArrowSchema schema = {0}; | |
| check_status(library->schema(source, &schema)); | |
| CHECK(schema.release != NULL && schema.n_children == 2, "Unexpected source schema"); | |
| CHECK(strcmp(schema.format, "+s") == 0, "The schema must describe a struct"); | |
| CHECK(strcmp(schema.children[0]->name, "id") == 0, "Missing id field"); | |
| CHECK(strcmp(schema.children[0]->format, "l") == 0, "id must be int64"); | |
| CHECK(strcmp(schema.children[1]->name, "label") == 0, "Missing label field"); | |
| CHECK(strcmp(schema.children[1]->format, "u") == 0, "label must be UTF-8"); | |
| SparkDsReaderInfo info = {.size = sizeof(SparkDsReaderInfo)}; | |
| void* reader = NULL; | |
| check_status(library->create_reader(source, &schema, &info, &reader)); | |
| CHECK(reader != NULL, "create_reader returned a null reader"); | |
| release_input(&schema); | |
| library->close(source); | |
| return reader; | |
| } | |
| static Plan finish_plan(void* reader) { | |
| Plan plan = {0}; | |
| SparkDsPartitionsInfo info = {.size = sizeof(SparkDsPartitionsInfo)}; | |
| check_status(library->partitions(reader, &info, add_partition, set_state, &plan)); | |
| CHECK(plan.states.length == 1, "partitions must set exactly one state"); | |
| library->close_reader(reader); | |
| return plan; | |
| } | |
| static Plan plan_scan( | |
| const char* const* keys, | |
| const char* const* values, | |
| int64_t option_count, | |
| int32_t limit, | |
| const char* const* projection, | |
| int64_t projection_count) { | |
| void* reader = create_planner(keys, values, option_count); | |
| /* Spark calls each pushdown function at most once, in this order. */ | |
| if (limit >= 0) { | |
| bool applied = false; | |
| check_status(library->push_limit(reader, limit, &applied)); | |
| CHECK(applied, "The Range source must apply its global limit"); | |
| } | |
| check_status(library->prune_columns(reader, projection, projection_count)); | |
| return finish_plan(reader); | |
| } | |
| static struct ArrowArrayStream open_stream( | |
| const Plan* plan, | |
| int32_t kind, | |
| size_t partition, | |
| const char* const* columns, | |
| int64_t column_count, | |
| const SparkDsTaskContext* task) { | |
| struct ArrowArrayStream stream = {0}; | |
| struct ArrowSchema schema = read_schema(columns, column_count); | |
| const Bytes* state = &plan->states.values[0]; | |
| const Bytes* bytes = &plan->partitions.values[partition]; | |
| SparkDsReadInfo info = { | |
| .size = sizeof(SparkDsReadInfo), | |
| .kind = kind, | |
| .schema = &schema, | |
| .reader_state = state->data, | |
| .reader_state_length = state->length, | |
| .partition = bytes->data, | |
| .partition_length = bytes->length | |
| }; | |
| check_status(library->read(task, &info, &stream)); | |
| release_input(&schema); | |
| CHECK(stream.release != NULL && stream.get_schema != NULL && stream.get_next != NULL, | |
| "read returned an incomplete Arrow stream"); | |
| return stream; | |
| } | |
| static void check_batch( | |
| const struct ArrowSchema* schema, | |
| const struct ArrowArray* batch, | |
| int64_t first_id, | |
| int64_t max_batch_size) { | |
| CHECK(batch->offset == 0, "The root array must have zero offset"); | |
| CHECK(batch->length > 0 && batch->length <= max_batch_size, "Unexpected batch row count"); | |
| CHECK(batch->n_children == schema->n_children, "Batch and schema have different column counts"); | |
| for (int64_t column = 0; column < schema->n_children; column++) { | |
| const struct ArrowSchema* field = schema->children[column]; | |
| const struct ArrowArray* array = batch->children[column]; | |
| CHECK(array->offset == 0 && array->length == batch->length, "Unexpected child array layout"); | |
| if (strcmp(field->name, "id") == 0) { | |
| CHECK(strcmp(field->format, "l") == 0, "Unexpected id type"); | |
| const int64_t* ids = array->buffers[1]; | |
| for (int64_t row = 0; row < batch->length; row++) { | |
| CHECK(ids[row] == first_id + row, "Unexpected id value"); | |
| } | |
| } else { | |
| CHECK(strcmp(field->name, "label") == 0, "Unexpected output field"); | |
| CHECK(strcmp(field->format, "u") == 0, "Unexpected label type"); | |
| const int32_t* offsets = array->buffers[1]; | |
| const char* data = array->buffers[2]; | |
| for (int64_t row = 0; row < batch->length; row++) { | |
| char expected[64]; | |
| int length = snprintf(expected, sizeof(expected), "row-%" PRId64, first_id + row); | |
| CHECK(offsets[row + 1] - offsets[row] == length, "Unexpected label length"); | |
| CHECK(memcmp(data + offsets[row], expected, (size_t) length) == 0, | |
| "Unexpected label value"); | |
| } | |
| } | |
| } | |
| } | |
| static void consume_plan( | |
| const Plan* plan, | |
| int32_t kind, | |
| int64_t first_id, | |
| int64_t expected_rows, | |
| const char* const* columns, | |
| int64_t column_count) { | |
| int64_t rows = 0; | |
| for (size_t partition = 0; partition < plan->partitions.length; partition++) { | |
| SparkDsTaskContext task = task_context(); | |
| expected_task = &task; | |
| interrupted = false; | |
| struct ArrowArrayStream stream = | |
| open_stream(plan, kind, partition, columns, column_count, &task); | |
| struct ArrowSchema schema = {0}; | |
| CHECK(stream.get_schema(&stream, &schema) == 0, "Cannot read the stream schema"); | |
| CHECK(schema.n_children == column_count, "Unexpected projected column count"); | |
| for (int64_t i = 0; i < column_count; i++) { | |
| CHECK(strcmp(schema.children[i]->name, columns[i]) == 0, | |
| "The stream does not have the columns of the schema to read, in order"); | |
| } | |
| struct ArrowArray* batches = NULL; | |
| size_t count = 0; | |
| while (true) { | |
| struct ArrowArray batch = {0}; | |
| int status = stream.get_next(&stream, &batch); | |
| if (status != 0) { | |
| fprintf(stderr, "Arrow error: %s\n", stream.get_last_error(&stream)); | |
| } | |
| CHECK(status == 0, "Cannot read a batch"); | |
| if (batch.release == NULL) { | |
| break; | |
| } | |
| struct ArrowArray* resized = realloc(batches, (count + 1) * sizeof(struct ArrowArray)); | |
| CHECK(resized != NULL, "Cannot retain batches"); | |
| batches = resized; | |
| batches[count++] = batch; | |
| } | |
| stream.release(&stream); | |
| CHECK(stream.release == NULL, "Releasing the stream must clear its release callback"); | |
| /* The arrays retain their buffers across later reads and the release of their stream. */ | |
| for (size_t i = 0; i < count; i++) { | |
| check_batch(&schema, &batches[i], first_id + rows, 2); | |
| rows += batches[i].length; | |
| batches[i].release(&batches[i]); | |
| } | |
| free(batches); | |
| schema.release(&schema); | |
| } | |
| CHECK(rows == expected_rows, "Unexpected total row count"); | |
| } | |
| /* A NUL-terminated copy of a value that a sink received, such as an offset. */ | |
| static char* copy_string(const Bytes* bytes) { | |
| char* string = malloc((size_t) bytes->length + 1); | |
| CHECK(string != NULL, "Cannot allocate a string"); | |
| memcpy(string, bytes->data, (size_t) bytes->length); | |
| string[bytes->length] = '\0'; | |
| return string; | |
| } | |
| static void* create_stream_reader( | |
| const char* const* keys, const char* const* values, int64_t count) { | |
| void* source = create_source(keys, values, count); | |
| struct ArrowSchema schema = {0}; | |
| check_status(library->schema(source, &schema)); | |
| SparkDsStreamReaderInfo info = { | |
| .size = sizeof(SparkDsStreamReaderInfo), | |
| .checkpoint_location = "file:/tmp/checkpoint/sources/0" | |
| }; | |
| void* stream = NULL; | |
| check_status(library->create_stream_reader(source, &schema, &info, &stream)); | |
| CHECK(stream != NULL, "create_stream_reader returned a null reader"); | |
| release_input(&schema); | |
| /* Spark closes the source right after it creates the stream reader. */ | |
| library->close(source); | |
| return stream; | |
| } | |
| /* The latest offset after start, within a limit and the end of a run, which may be NULL. */ | |
| static char* latest_offset( | |
| void* stream, const char* start, const SparkDsReadLimit* limit, const char* run_end) { | |
| SparkDsLatestOffsetInfo info = { | |
| .size = sizeof(SparkDsLatestOffsetInfo), | |
| .start = start, | |
| .limit = limit, | |
| .run_end = run_end | |
| }; | |
| ByteList offsets = {0}; | |
| check_status(library->latest_offset(stream, &info, add_bytes, &offsets)); | |
| CHECK(offsets.length == 1, "latest_offset must emit exactly one offset"); | |
| char* offset = copy_string(&offsets.values[0]); | |
| free_list(&offsets); | |
| return offset; | |
| } | |
| static Plan plan_micro_batch(void* stream, const char* start, const char* end) { | |
| Plan plan = {0}; | |
| SparkDsStreamPartitionsInfo info = { | |
| .size = sizeof(SparkDsStreamPartitionsInfo), | |
| .start = start, | |
| .end = end | |
| }; | |
| check_status(library->stream_partitions(stream, &info, add_partition, set_state, &plan)); | |
| CHECK(plan.states.length == 1, "stream_partitions must set exactly one state"); | |
| return plan; | |
| } | |
| /* | |
| * A streaming read: the stream reader stays on the driver, and plans each micro-batch between | |
| * two offsets that the library defines as JSON. The executors read a micro-batch with the read | |
| * function of batch scans. | |
| */ | |
| static void check_micro_batch_read( | |
| const char* const* keys, const char* const* values, int64_t count) { | |
| void* stream = create_stream_reader(keys, values, count); | |
| SparkDsReadLimit limit = {.size = sizeof(SparkDsReadLimit), .kind = SPARK_DS_READ_ALL_AVAILABLE}; | |
| check_status(library->default_read_limit(stream, &limit)); | |
| CHECK(limit.kind == SPARK_DS_READ_ALL_AVAILABLE, "The Range source has no limit by default"); | |
| SparkDsInitialOffsetInfo initial_info = {.size = sizeof(SparkDsInitialOffsetInfo)}; | |
| ByteList offsets = {0}; | |
| check_status(library->initial_offset(stream, &initial_info, add_bytes, &offsets)); | |
| CHECK(offsets.length == 1, "initial_offset must emit exactly one offset"); | |
| char* initial = copy_string(&offsets.values[0]); | |
| free_list(&offsets); | |
| CHECK(strcmp(initial, "{\"next\":10}") == 0, "Unexpected initial offset"); | |
| char* latest = latest_offset(stream, initial, &limit, NULL); | |
| CHECK(strcmp(latest, "{\"next\":25}") == 0, "Unexpected latest offset"); | |
| /* Two micro-batches, split at an offset as if from the checkpoint of an earlier query. */ | |
| const char* columns[] = {"id", "label"}; | |
| Plan first = plan_micro_batch(stream, initial, "{\"next\":13}"); | |
| consume_plan(&first, SPARK_DS_READ_MICRO_BATCH, 10, 3, columns, 2); | |
| free_plan(&first); | |
| Plan second = plan_micro_batch(stream, "{\"next\":13}", latest); | |
| consume_plan(&second, SPARK_DS_READ_MICRO_BATCH, 13, 12, columns, 2); | |
| free_plan(&second); | |
| SparkDsCommitOffsetInfo commit_info = {.size = sizeof(SparkDsCommitOffsetInfo), .end = latest}; | |
| check_status(library->commit_offset(stream, &commit_info)); | |
| library->close_stream_reader(stream); | |
| /* An offset that does not decode fails its call, after which Spark closes the reader. */ | |
| stream = create_stream_reader(keys, values, count); | |
| Plan failed = {0}; | |
| SparkDsStreamPartitionsInfo invalid = { | |
| .size = sizeof(SparkDsStreamPartitionsInfo), | |
| .start = initial, | |
| .end = "not an offset" | |
| }; | |
| CHECK(library->stream_partitions(stream, &invalid, add_partition, set_state, &failed) != 0, | |
| "An offset that does not decode must fail"); | |
| CHECK(failed.partitions.length == 0 && failed.states.length == 0, | |
| "A failed plan must not return partitions or a state"); | |
| const char* error = library->last_error(); | |
| CHECK(error != NULL && strstr(error, "cannot decode the offset") != NULL, | |
| "Missing offset error"); | |
| library->close_stream_reader(stream); | |
| free(initial); | |
| free(latest); | |
| } | |
| /* Admission control: each micro-batch reads at most maxRowsPerTrigger rows. */ | |
| static void check_admission_control(void) { | |
| const char* keys[] = {"start", "end", "maxrowspertrigger"}; | |
| const char* values[] = {"10", "25", "4"}; | |
| void* stream = create_stream_reader(keys, values, 3); | |
| SparkDsReadLimit limit = {.size = sizeof(SparkDsReadLimit), .kind = SPARK_DS_READ_ALL_AVAILABLE}; | |
| check_status(library->default_read_limit(stream, &limit)); | |
| CHECK(limit.kind == SPARK_DS_READ_MAX_ROWS && limit.max_rows == 4, "Unexpected read limit"); | |
| char* first = latest_offset(stream, "{\"next\":10}", &limit, NULL); | |
| CHECK(strcmp(first, "{\"next\":14}") == 0, "The limit must end the micro-batch"); | |
| /* With Trigger.AvailableNow, the latest offset does not go beyond the end of the run. */ | |
| char* last = latest_offset(stream, "{\"next\":14}", &limit, "{\"next\":16}"); | |
| CHECK(strcmp(last, "{\"next\":16}") == 0, "The end of the run must end the micro-batch"); | |
| SparkDsReadLimit all = {.size = sizeof(SparkDsReadLimit), .kind = SPARK_DS_READ_ALL_AVAILABLE}; | |
| char* latest = latest_offset(stream, "{\"next\":14}", &all, NULL); | |
| CHECK(strcmp(latest, "{\"next\":25}") == 0, "Without a limit, all the rows are available"); | |
| free(first); | |
| free(last); | |
| free(latest); | |
| library->close_stream_reader(stream); | |
| } | |
| static void check_independent_planners( | |
| const char* const* keys, const char* const* values, int64_t count) { | |
| void* first = create_planner(keys, values, count); | |
| void* second = create_planner(keys, values, count); | |
| bool applied = false; | |
| check_status(library->push_limit(first, 4, &applied)); | |
| CHECK(applied, "The first planner did not accept its limit"); | |
| const char* label_only[] = {"label"}; | |
| check_status(library->prune_columns(first, label_only, 1)); | |
| Plan first_plan = finish_plan(first); | |
| /* The second planner must neither inherit the first limit nor change its serialized plan. */ | |
| const char* id_only[] = {"id"}; | |
| check_status(library->prune_columns(second, id_only, 1)); | |
| Plan second_plan = finish_plan(second); | |
| /* Both driver planners are already closed; executor reads only get the copied bytes. */ | |
| consume_plan(&second_plan, SPARK_DS_READ_BATCH, 10, 15, id_only, 1); | |
| consume_plan(&first_plan, SPARK_DS_READ_BATCH, 10, 4, label_only, 1); | |
| free_plan(&second_plan); | |
| free_plan(&first_plan); | |
| } | |
| static void check_cancellation(const Plan* plan, const char* const* columns) { | |
| CHECK(plan->partitions.length == 1, "Cancellation test requires one partition"); | |
| SparkDsTaskContext task = task_context(); | |
| expected_task = &task; | |
| interrupted = false; | |
| interruption_checks = 0; | |
| struct ArrowArrayStream stream = open_stream(plan, SPARK_DS_READ_BATCH, 0, columns, 2, &task); | |
| struct ArrowSchema schema = {0}; | |
| CHECK(stream.get_schema(&stream, &schema) == 0, "Cannot read the stream schema"); | |
| struct ArrowArray first = {0}; | |
| CHECK(stream.get_next(&stream, &first) == 0 && first.release != NULL, "Missing first batch"); | |
| CHECK(interruption_checks > 0, "The reader did not check the task callback"); | |
| int previous_checks = interruption_checks; | |
| interrupted = true; | |
| struct ArrowArray next = {0}; | |
| CHECK(stream.get_next(&stream, &next) != 0, | |
| "A cancelled task must fail before its next batch"); | |
| CHECK(interruption_checks > previous_checks, "Cancellation did not call the task callback"); | |
| const char* error = stream.get_last_error(&stream); | |
| CHECK(error != NULL && strstr(error, "cancel") != NULL, "Missing cancellation error"); | |
| if (next.release != NULL) { | |
| next.release(&next); | |
| } | |
| stream.release(&stream); | |
| /* Early stream release must not invalidate the batch already handed to its consumer. */ | |
| check_batch(&schema, &first, 0, 2); | |
| first.release(&first); | |
| schema.release(&schema); | |
| interrupted = false; | |
| } | |
| enum StreamOperation { | |
| STREAM_SCHEMA, | |
| STREAM_NEXT, | |
| STREAM_ERROR, | |
| STREAM_RELEASE | |
| }; | |
| typedef struct StreamCall { | |
| struct ArrowArrayStream* stream; | |
| enum StreamOperation operation; | |
| struct ArrowSchema schema; | |
| struct ArrowArray batch; | |
| int status; | |
| char error[256]; | |
| } StreamCall; | |
| static void* run_stream_call(void* argument) { | |
| StreamCall* call = argument; | |
| switch (call->operation) { | |
| case STREAM_SCHEMA: | |
| call->status = call->stream->get_schema(call->stream, &call->schema); | |
| break; | |
| case STREAM_NEXT: | |
| call->status = call->stream->get_next(call->stream, &call->batch); | |
| break; | |
| case STREAM_ERROR: { | |
| const char* error = call->stream->get_last_error(call->stream); | |
| CHECK(error != NULL, "A stream error must remain available on another thread"); | |
| snprintf(call->error, sizeof(call->error), "%s", error); | |
| break; | |
| } | |
| case STREAM_RELEASE: | |
| call->stream->release(call->stream); | |
| break; | |
| } | |
| return NULL; | |
| } | |
| static void stream_call_on_thread(StreamCall* call) { | |
| pthread_t thread; | |
| CHECK(pthread_create(&thread, NULL, run_stream_call, call) == 0, | |
| "Cannot create a stream callback thread"); | |
| CHECK(pthread_join(thread, NULL) == 0, "Cannot join a stream callback thread"); | |
| } | |
| /* Every host has the fields of version 1: a shorter structure is an error, not partly read. */ | |
| static void check_short_structures(const Plan* plan, const char* const* columns) { | |
| SparkDsTaskContext task = task_context(); | |
| task.size = offsetof(SparkDsTaskContext, is_interrupted); | |
| task.is_interrupted = unavailable_callback; | |
| struct ArrowSchema schema = read_schema(columns, 2); | |
| const Bytes* state = &plan->states.values[0]; | |
| const Bytes* bytes = &plan->partitions.values[0]; | |
| SparkDsReadInfo info = { | |
| .size = sizeof(SparkDsReadInfo), | |
| .kind = SPARK_DS_READ_BATCH, | |
| .schema = &schema, | |
| .reader_state = state->data, | |
| .reader_state_length = state->length, | |
| .partition = bytes->data, | |
| .partition_length = bytes->length | |
| }; | |
| struct ArrowArrayStream stream = {0}; | |
| CHECK(library->read(&task, &info, &stream) != 0, | |
| "A task context shorter than version 1 must be rejected"); | |
| const char* error = library->last_error(); | |
| CHECK(error != NULL && strstr(error, "version 1") != NULL, "Missing task context error"); | |
| CHECK(stream.release == NULL, "A failed read must not initialize the stream"); | |
| release_input(&schema); | |
| task = task_context(); | |
| schema = read_schema(columns, 2); | |
| info.size = offsetof(SparkDsReadInfo, partition); | |
| CHECK(library->read(&task, &info, &stream) != 0, | |
| "A read info shorter than version 1 must be rejected"); | |
| error = library->last_error(); | |
| CHECK(error != NULL && strstr(error, "SparkDsReadInfo") != NULL, "Missing read info error"); | |
| CHECK(stream.release == NULL, "A failed read must not initialize the stream"); | |
| release_input(&schema); | |
| } | |
| static void check_cross_thread_stream(const Plan* plan, const char* const* columns) { | |
| SparkDsTaskContext task = task_context(); | |
| expected_task = &task; | |
| interrupted = false; | |
| interruption_checks = 0; | |
| struct ArrowArrayStream stream = open_stream(plan, SPARK_DS_READ_BATCH, 0, columns, 2, &task); | |
| /* Callbacks do not overlap, but each worker has a different lifetime and thread-local state. */ | |
| StreamCall schema = {.stream = &stream, .operation = STREAM_SCHEMA}; | |
| stream_call_on_thread(&schema); | |
| CHECK(schema.status == 0 && schema.schema.release != NULL, "Cannot get schema on a new thread"); | |
| StreamCall first = {.stream = &stream, .operation = STREAM_NEXT}; | |
| stream_call_on_thread(&first); | |
| CHECK(first.status == 0 && first.batch.release != NULL, "Cannot read a batch on a new thread"); | |
| CHECK(interruption_checks > 0, "The worker did not call the task cancellation callback"); | |
| interrupted = true; | |
| StreamCall next = {.stream = &stream, .operation = STREAM_NEXT}; | |
| stream_call_on_thread(&next); | |
| CHECK(next.status != 0, "The worker must observe live task cancellation"); | |
| StreamCall error = {.stream = &stream, .operation = STREAM_ERROR}; | |
| stream_call_on_thread(&error); | |
| CHECK(strstr(error.error, "cancel") != NULL, "The stream error changed across threads"); | |
| if (next.batch.release != NULL) { | |
| next.batch.release(&next.batch); | |
| } | |
| StreamCall release = {.stream = &stream, .operation = STREAM_RELEASE}; | |
| stream_call_on_thread(&release); | |
| CHECK(stream.release == NULL, "Cross-thread release must clear the stream callback"); | |
| check_batch(&schema.schema, &first.batch, 0, 2); | |
| first.batch.release(&first.batch); | |
| schema.schema.release(&schema.schema); | |
| interrupted = false; | |
| } | |
| static void check_error_message(void) { | |
| const char* keys[] = {"batchsize"}; | |
| const char* values[] = {"0"}; | |
| SparkDsCreateInfo info = { | |
| .size = sizeof(SparkDsCreateInfo), | |
| .name = "rust_sdk_range", | |
| .option_keys = keys, | |
| .option_values = values, | |
| .num_options = 1 | |
| }; | |
| uint64_t capabilities = 0; | |
| void* source = NULL; | |
| CHECK(library->create(&info, &capabilities, &source) != 0, | |
| "Invalid options must return an error"); | |
| const char* error = library->last_error(); | |
| CHECK(error != NULL && strstr(error, "must be positive") != NULL, | |
| "The C error channel did not preserve the provider's message"); | |
| CHECK(source == NULL, "Failed creation must not return a source handle"); | |
| } | |
| static void check_function_table(void) { | |
| CHECK(library != NULL, "spark_ds_init returned no library"); | |
| CHECK(library->size == sizeof(SparkDsLibrary), "The function table differs from Spark's header"); | |
| CHECK(library->abi_version == SPARK_DS_ABI_VERSION, "Unexpected ABI version"); | |
| CHECK(library->last_error != NULL, "Missing last_error function"); | |
| CHECK(library->create != NULL && library->schema != NULL && library->close != NULL, | |
| "Missing source functions"); | |
| CHECK(library->create_reader != NULL && library->push_limit != NULL && | |
| library->prune_columns != NULL && library->partitions != NULL && | |
| library->close_reader != NULL && library->read != NULL, "Missing batch read functions"); | |
| CHECK(library->create_stream_reader != NULL && library->initial_offset != NULL && | |
| library->latest_offset != NULL && library->stream_partitions != NULL && | |
| library->commit_offset != NULL && library->close_stream_reader != NULL && | |
| library->default_read_limit != NULL, "Missing micro-batch read functions"); | |
| CHECK(library->create_writer == NULL && library->commit == NULL && library->abort == NULL && | |
| library->close_writer == NULL && library->create_data_writer == NULL && | |
| library->write == NULL && library->commit_data_writer == NULL && | |
| library->abort_data_writer == NULL && library->close_data_writer == NULL, | |
| "A read-only source must not export writes"); | |
| } | |
| int main(int argc, char** argv) { | |
| CHECK(argc == 2, "Expected the path to the Range shared library"); | |
| void* module = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); | |
| if (module == NULL) { | |
| fprintf(stderr, "Cannot load the Range library: %s\n", dlerror()); | |
| return 1; | |
| } | |
| void* symbol = dlsym(module, "spark_ds_init"); | |
| CHECK(symbol != NULL, "The library does not export spark_ds_init"); | |
| int (*init)(const SparkDsHost*, const SparkDsLibrary**); | |
| CHECK(sizeof(init) == sizeof(symbol), "This platform has incompatible function pointers"); | |
| memcpy(&init, &symbol, sizeof(init)); | |
| SparkDsHost host = { | |
| .size = sizeof(SparkDsHost), | |
| .abi_version = SPARK_DS_ABI_VERSION, | |
| .spark_version = "4.4.0-c-abi-test", | |
| .log = log_message, | |
| .is_log_enabled = is_log_enabled | |
| }; | |
| CHECK(init(&host, &library) == 0, "spark_ds_init failed"); | |
| check_function_table(); | |
| const char* keys[] = {"start", "end", "numpartitions", "batchsize"}; | |
| const char* values[] = {"10", "25", "3", "2"}; | |
| const char* reordered[] = {"label", "id"}; | |
| const char* id_only[] = {"id"}; | |
| const char* original[] = {"id", "label"}; | |
| check_capability_discovery(keys, values, 4); | |
| check_capability_discovery(NULL, NULL, 0); | |
| check_independent_planners(keys, values, 4); | |
| check_micro_batch_read(keys, values, 4); | |
| check_admission_control(); | |
| Plan projected = plan_scan(keys, values, 4, -1, reordered, 2); | |
| consume_plan(&projected, SPARK_DS_READ_BATCH, 10, 15, reordered, 2); | |
| free_plan(&projected); | |
| Plan limited = plan_scan(keys, values, 4, 5, id_only, 1); | |
| consume_plan(&limited, SPARK_DS_READ_BATCH, 10, 5, id_only, 1); | |
| free_plan(&limited); | |
| const char* short_keys[] = {"end", "numpartitions", "batchsize"}; | |
| const char* short_values[] = {"7", "2", "2"}; | |
| Plan empty = plan_scan(short_keys, short_values, 3, -1, NULL, 0); | |
| consume_plan(&empty, SPARK_DS_READ_BATCH, 0, 7, NULL, 0); | |
| free_plan(&empty); | |
| const char* one_partition[] = {"20", "1", "2"}; | |
| Plan cancellable = plan_scan(short_keys, one_partition, 3, -1, original, 2); | |
| consume_plan(&cancellable, SPARK_DS_READ_BATCH, 0, 20, original, 2); | |
| check_short_structures(&cancellable, original); | |
| check_cancellation(&cancellable, original); | |
| check_cross_thread_stream(&cancellable, original); | |
| free_plan(&cancellable); | |
| check_error_message(); | |
| CHECK(dlclose(module) == 0, "Cannot close the Range library"); | |
| puts("C ABI, capabilities, planning, streaming, admission control, Arrow ownership, and " | |
| "cancellation checks passed."); | |
| return 0; | |
| } |