| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| """Compare committed timestamp windows, including cross-snapshot change events.""" | |
| import json | |
| from contextlib import ExitStack | |
| from unittest.mock import patch | |
| import pyarrow as pa | |
| import pytest | |
| from pypaimon import CatalogFactory, Schema | |
| from pypaimon.common.identifier import Identifier | |
| from pypaimon.execution.native.runtime import (native_runtime_available) | |
| from pypaimon.execution.python.read.scan_distribution import java_file_name_shard | |
| from pypaimon.read.split import DataSplit | |
| from pypaimon.table.row.generic_row import GenericRow | |
| from pypaimon.table.row.row_kind import RowKind | |
| from pypaimon.utils.range import Range | |
| @pytest.fixture(params=[False, pytest.param( | |
| True, marks=[pytest.mark.native_plan, pytest.mark.skipif( | |
| not native_runtime_available(), | |
| reason='pypaimon-rust not installed')])], | |
| ids=['python', 'native']) | |
| def native(request): | |
| return request.param | |
| @pytest.fixture | |
| def catalog(tmp_path): | |
| result = CatalogFactory.create({'warehouse': str(tmp_path)}) | |
| result.create_database('default', True) | |
| return result | |
| SCHEMA = pa.schema([('k', pa.int64()), ('v', pa.string())]) | |
| def _table(catalog, name, primary_key=False, options=None): | |
| catalog.create_table('default.' + name, Schema.from_pyarrow_schema( | |
| SCHEMA, primary_keys=['k'] if primary_key else None, | |
| options=options), False) | |
| return catalog.get_table('default.' + name) | |
| def _set_time(table, timestamp): | |
| manager = table.snapshot_manager() | |
| snapshot = manager.get_latest_snapshot() | |
| path = manager.get_snapshot_path(snapshot.id) | |
| with open(path) as reader: | |
| data = json.load(reader) | |
| data['timeMillis'] = timestamp | |
| with open(path, 'w') as writer: | |
| json.dump(data, writer) | |
| return snapshot.id | |
| def _write(table, timestamp, rows, overwrite=False, kind=None): | |
| builder = table.new_batch_write_builder() | |
| if overwrite: | |
| builder.overwrite() | |
| writer, commit = builder.new_write(), builder.new_commit() | |
| try: | |
| if kind is None: | |
| writer.write_arrow(pa.Table.from_pydict({ | |
| key: [row[key] for row in rows] for key in SCHEMA.names}, schema=SCHEMA)) | |
| else: | |
| for row in rows: | |
| writer.write_row(GenericRow( | |
| [row[field.name] for field in table.fields], table.fields, kind)) | |
| commit.commit(writer.prepare_commit()) | |
| finally: | |
| writer.close() | |
| commit.close() | |
| return _set_time(table, timestamp) | |
| def _read(table, native, window, predicate=None, limit=None, shard=None, | |
| slice_=None, row_ranges=None, with_stats=False): | |
| builder = table.copy({ | |
| 'scan.native-plan.enabled': str(native).lower(), | |
| 'read.native.enabled': str(native).lower(), | |
| 'scan.mode': 'incremental', | |
| 'incremental-between-timestamp': '%s,%s' % window, | |
| }).new_read_builder() | |
| if predicate is not None: | |
| builder.with_filter(predicate) | |
| if limit is not None: | |
| builder.with_limit(limit) | |
| scan = builder.new_scan() | |
| if shard is not None: | |
| scan.with_shard(*shard) | |
| if slice_ is not None: | |
| scan.with_slice(*slice_) | |
| if row_ranges is not None: | |
| scan.with_row_ranges(row_ranges) | |
| with ExitStack() as stack: | |
| if native: | |
| stack.enter_context(patch.object( | |
| scan, '_create_file_scanner', side_effect=AssertionError( | |
| 'incremental native plan fell back to Python'))) | |
| plan = scan.scan_with_stats()[0] if with_stats else scan.plan() | |
| with ExitStack() as stack: | |
| if native: | |
| assert all( | |
| getattr(split, '_native_split', None) is not None | |
| for split in plan.splits()) | |
| stack.enter_context(patch( | |
| 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', | |
| side_effect=AssertionError( | |
| 'incremental native read fell back to Python'))) | |
| result = builder.new_read().to_arrow( | |
| plan.splits(), parallelism=1).to_pydict() | |
| return plan, [dict(zip(result, row)) for row in zip(*result.values())] | |
| @pytest.fixture | |
| def history(catalog): | |
| table = _table(catalog, 'history', True, {'bucket': '1'}) | |
| _write(table, 100, [{'k': 1, 'v': 'base'}, {'k': 2, 'v': 'base'}]) | |
| _write(table, 200, [{'k': 1, 'v': 'intermediate'}, {'k': 3, 'v': 'third'}]) | |
| _write(table, 200, [{'k': 1, 'v': 'latest'}, {'k': 4, 'v': 'fourth'}]) | |
| _write(table, 300, [{'k': 9, 'v': 'overwrite'}], overwrite=True) | |
| assert table.snapshot_manager().get_latest_snapshot().commit_kind == 'OVERWRITE' | |
| _write(table, 400, [{'k': 5, 'v': 'fifth'}]) | |
| return table | |
| @pytest.mark.parametrize('window,snapshot_id,expected', [ | |
| ((100, 200), 3, [(1, 'intermediate'), (1, 'latest'), (3, 'third'), (4, 'fourth')]), | |
| ((200, 300), 4, []), | |
| ((100, 300), 4, [(1, 'intermediate'), (1, 'latest'), (3, 'third'), (4, 'fourth')]), | |
| ((100, 150), 1, []), | |
| ((500, 600), None, []), | |
| ((0, 50), None, []), | |
| ((0, 100), 1, [(1, 'base'), (2, 'base')]), | |
| ((200, 400), 5, [(5, 'fifth')]), | |
| ]) | |
| def test_timestamp_windows_preserve_append_events_and_end_snapshot( | |
| native, history, window, snapshot_id, expected): | |
| plan, rows = _read(history, native, window) | |
| assert plan.snapshot_id == snapshot_id | |
| assert sorted((row['k'], row['v']) for row in rows) == expected | |
| assert all(split.snapshot_id == snapshot_id for split in plan.splits()) | |
| def test_predicate_can_select_an_earlier_event(native, history): | |
| predicate = history.new_read_builder().new_predicate_builder().equal('v', 'intermediate') | |
| plan, rows = _read(history, native, (100, 200), predicate=predicate, with_stats=True) | |
| assert plan.snapshot_id == 3 | |
| assert all(split.snapshot_id == plan.snapshot_id for split in plan.splits()) | |
| assert rows == [{'k': 1, 'v': 'intermediate'}] | |
| def test_append_distribution_precedes_limit_in_incremental_window(native, catalog): | |
| table = _table(catalog, 'append', options={'source.split.target-size': '1b'}) | |
| rows = [{'k': key, 'v': str(key)} for key in range(12)] | |
| for offset in range(0, 12, 3): | |
| _write(table, (offset // 3 + 1) * 100, rows[offset:offset + 3]) | |
| for options in ({'shard': (1, 3)}, {'slice_': (3, 8)}): | |
| plan, actual = _read(table, native, (100, 400), limit=2, **options) | |
| assert plan.snapshot_id == 4 | |
| assert actual == rows[6:8] | |
| def test_primary_key_shards_preserve_all_selected_commits(native, catalog): | |
| table = _table(catalog, 'pk_shards', True, {'bucket': '4'}) | |
| rows = [{'k': key, 'v': 'base'} for key in range(20)] | |
| _write(table, 100, rows) | |
| _write(table, 200, [dict(row, v='old') for row in rows]) | |
| _write(table, 300, [dict(row, v='new') for row in rows]) | |
| result = [] | |
| for shard in range(4): | |
| plan, actual = _read(table, native, (100, 300), shard=(shard, 4)) | |
| assert plan.snapshot_id == 3 | |
| assert all(split.bucket % 4 == shard for split in plan.splits()) | |
| result.extend(actual) | |
| if actual: | |
| _, limited = _read(table, native, (100, 300), shard=(shard, 4), limit=1) | |
| assert len(limited) == 1 | |
| assert limited[0] in actual | |
| assert sorted((row['k'], row['v']) for row in result) == [ | |
| (key, version) for key in range(20) for version in ('new', 'old')] | |
| @pytest.mark.parametrize('engine,dv', [('first-row', False), ('deduplicate', True)]) | |
| def test_raw_primary_key_incremental_shards_preserve_file_events(native, catalog, engine, dv): | |
| table = _table(catalog, 'raw_pk_shards', True, { | |
| 'bucket': '1', 'merge-engine': engine, | |
| 'deletion-vectors.enabled': str(dv).lower(), | |
| 'source.split.target-size': '1b', | |
| }) | |
| _write(table, 100, [{'k': 0, 'v': 'outside'}]) | |
| for key in range(8): | |
| _write(table, 200 + key * 2, [{'k': key, 'v': 'old'}]) | |
| _write(table, 201 + key * 2, [{'k': key, 'v': 'new'}]) | |
| events, file_names = [], set() | |
| for shard in range(5): | |
| plan, rows = _read(table, native, (100, 400), shard=(shard, 5)) | |
| assert plan.snapshot_id == 17 | |
| assert all(split.is_streaming and split.bucket == 0 for split in plan.splits()) | |
| names = {file.file_name for split in plan.splits() for file in split.files} | |
| assert all(java_file_name_shard(name, 5) == shard for name in names) | |
| assert not file_names.intersection(names) | |
| file_names.update(names) | |
| events.extend(rows) | |
| if rows: | |
| _, limited = _read(table, native, (100, 400), shard=(shard, 5), limit=1) | |
| assert len(limited) == 1 and limited[0] in rows | |
| assert len(file_names) == 16 | |
| assert sorted((row['k'], row['v']) for row in events) == [ | |
| (key, version) for key in range(8) for version in ('new', 'old')] | |
| def test_data_evolution_positions_intersect_incremental_row_ranges(native, catalog): | |
| table = _table(catalog, 'de', options={ | |
| 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', | |
| 'source.split.target-size': '1b', | |
| }) | |
| rows = [{'k': key, 'v': str(key)} for key in range(9)] | |
| for offset in range(0, 9, 3): | |
| _write(table, (offset // 3 + 1) * 100, rows[offset:offset + 3]) | |
| plan, actual = _read(table, native, (100, 300), slice_=(1, 5), | |
| row_ranges=[Range(5, 7)], limit=2) | |
| assert plan.snapshot_id == 3 | |
| assert actual == rows[5:7] | |
| for selection in ({'slice_': (3, 6)}, {'shard': (1, 2)}): | |
| plan, actual = _read(table, native, (100, 300), | |
| row_ranges=[Range(7, 7)], **selection) | |
| assert plan.snapshot_id == 3 | |
| assert actual == rows[7:8] | |
| _, actual = _read(table, native, (100, 300), slice_=(0, 1), | |
| row_ranges=[Range(5, 7)]) | |
| assert actual == [] | |
| def test_incremental_branch_uses_its_own_snapshot_history(native, catalog): | |
| table = _table(catalog, 'branch') | |
| _write(table, 100, [{'k': 1, 'v': 'base'}]) | |
| table.create_tag('base') | |
| catalog.create_branch(table.identifier, 'test', tag_name='base') | |
| branch = catalog.get_table(Identifier('default', 'branch', branch='test')) | |
| _write(branch, 200, [{'k': 2, 'v': 'branch'}]) | |
| _write(table, 300, [{'k': 3, 'v': 'main'}]) | |
| plan, actual = _read(branch, native, (100, 300)) | |
| assert plan.snapshot_id == 2 | |
| assert actual == [{'k': 2, 'v': 'branch'}] | |
| def test_incremental_ignores_deletion_vectors_from_window_end(native, catalog): | |
| table = _table(catalog, 'deletions', options={ | |
| 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', | |
| 'deletion-vectors.enabled': 'true', 'index-file-in-data-file-dir': 'true', | |
| }) | |
| rows = [{'k': key, 'v': str(key)} for key in range(6)] | |
| _write(table, 100, rows[:3]) | |
| _write(table, 200, rows[3:]) | |
| builder = table.new_batch_write_builder() | |
| messages = builder.new_update().delete_by_row_id([1, 4]) | |
| commit = builder.new_commit() | |
| try: | |
| commit.commit(messages) | |
| finally: | |
| commit.close() | |
| assert _set_time(table, 300) == 3 | |
| historical, actual = _read(table, native, (100, 200)) | |
| assert historical.snapshot_id == 2 | |
| assert actual == rows[3:] | |
| current, actual = _read(table, native, (100, 300)) | |
| assert current.snapshot_id == 3 | |
| assert actual == rows[3:] | |
| _, actual = _read(table, native, (100, 300), slice_=(1, 3), limit=1) | |
| assert actual == [rows[4]] | |
| @pytest.mark.parametrize('window', ['100', 'one,200', '0,100,200']) | |
| def test_invalid_timestamp_window_is_rejected_even_for_empty_tables(catalog, window): | |
| table = _table(catalog, 'invalid') | |
| with pytest.raises(ValueError): | |
| table.copy({'incremental-between-timestamp': window}).new_read_builder().new_scan() | |
| @pytest.mark.parametrize('engine,dv', [ | |
| ('deduplicate', False), ('deduplicate', True), | |
| ('partial-update', False), ('first-row', False)]) | |
| def test_incremental_includes_l0_across_merge_engines(catalog, native, engine, dv): | |
| table = _table(catalog, 'engines', True, { | |
| 'bucket': '1', 'merge-engine': engine, | |
| 'deletion-vectors.enabled': str(dv).lower(), 'source.split.target-size': '1b'}) | |
| _write(table, 100, [{'k': 1, 'v': 'old'}]) | |
| _write(table, 200, [{'k': 1, 'v': 'new'}]) | |
| for value in (None, 'old', 'new'): | |
| predicate = table.new_read_builder().new_predicate_builder().equal('v', value) \ | |
| if value is not None else None | |
| plan, rows = _read(table, native, (0, 200), predicate=predicate, | |
| limit=1 if value else None) | |
| assert all(s.is_streaming and not s.data_deletion_files for s in plan.splits()) | |
| assert sorted(row['v'] for row in rows) == ([value] if value else ['new', 'old']) | |
| def test_incremental_does_not_evaluate_endpoint_global_indexes(catalog, native): | |
| table = _table(catalog, 'endpoint_index', options={ | |
| 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', | |
| 'deletion-vectors.enabled': 'true'}) | |
| _write(table, 100, [{'k': 1, 'v': 'old'}]) | |
| _write(table, 200, [{'k': 2, 'v': 'new'}]) | |
| path = table.snapshot_manager().get_snapshot_path(2) | |
| data = json.loads(table.file_io.read_file_utf8(path)) | |
| data['indexManifest'] = 'missing-endpoint-index' | |
| table.file_io.write_file(path, json.dumps(data), overwrite=True) | |
| predicate = table.new_read_builder().new_predicate_builder().equal('v', 'old') | |
| _, rows = _read(table, native, (0, 200), predicate=predicate) | |
| assert rows == [{'k': 1, 'v': 'old'}] | |
| def test_incremental_rejects_manifest_delete_before_reconciliation(catalog, native): | |
| from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan | |
| table = _table(catalog, 'invalid_delta', True) | |
| _write(table, 100, [{'k': 1, 'v': 'old'}]) | |
| _write(table, 200, [{'k': 1, 'v': 'new'}], overwrite=True) | |
| path = table.snapshot_manager().get_snapshot_path(2) | |
| data = json.loads(table.file_io.read_file_utf8(path)) | |
| data['commitKind'] = 'APPEND' | |
| table.file_io.write_file(path, json.dumps(data), overwrite=True) | |
| # Allow fallback here: both implementations must reject the malformed input. | |
| table = table.copy({'scan.native-plan.enabled': str(native).lower(), | |
| 'incremental-between-timestamp': '0,200'}) | |
| with pytest.raises(ValueError, match='only ADD'): | |
| table.new_read_builder().new_scan().plan() | |
| scan = AsyncStreamingTableScan(table, prefetch_enabled=False) | |
| with pytest.raises(ValueError, match='only ADD'): | |
| scan._create_delta_plan(table.snapshot_manager().get_snapshot_by_id(2)) | |
| def test_incremental_reader_preserves_all_physical_row_kinds(catalog, native): | |
| import pyarrow.parquet as pq | |
| from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan | |
| table = _table(catalog, 'row_kinds', True, {'bucket': '1'}) | |
| scan = AsyncStreamingTableScan(table, prefetch_enabled=False) | |
| expected = [] | |
| for kind in range(4): | |
| # Write each physical kind through the public API so committed files | |
| # and manifest metadata remain consistent for both reader paths. | |
| snapshot_id = _write(table, (kind + 1) * 100, [{'k': 1, 'v': str(kind)}], kind=RowKind(kind)) | |
| plan = scan._create_delta_plan(table.snapshot_manager().get_snapshot_by_id(snapshot_id)) | |
| file = plan.splits()[0].files[0] | |
| assert pq.read_table(file.file_path, columns=['_VALUE_KIND']).column(0).to_pylist() == [kind] | |
| assert file.file_size == table.file_io.get_file_size(file.file_path) | |
| rows = [(row.get_field(1), row.get_row_kind().value) | |
| for row in table.new_read_builder().new_read().to_iterator(plan.splits())] | |
| assert rows == [(str(kind), kind)] | |
| assert plan.snapshot_id == snapshot_id | |
| expected.append((str(kind), kind)) | |
| builder = table.copy({'scan.native-plan.enabled': str(native).lower(), | |
| 'read.native.enabled': str(native).lower(), | |
| 'incremental-between-timestamp': '0,400'}).new_read_builder() | |
| batch_scan = builder.new_scan() | |
| with ExitStack() as stack: | |
| if native: | |
| stack.enter_context(patch( | |
| 'pypaimon.execution.python.read.scanner.file_scanner.FileScanner.scan', | |
| side_effect=AssertionError('native fallback'))) | |
| plan = batch_scan.plan() | |
| # InternalRow views are reused by the iterator; capture values as it advances. | |
| rows = [(row.get_field(1), row.get_row_kind().value) | |
| for row in builder.new_read().to_iterator(plan.splits())] | |
| assert sorted(rows) == expected | |
| if native: | |
| read = builder.new_read() | |
| read.include_row_kind = True | |
| with patch( | |
| 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', | |
| side_effect=AssertionError('native row-kind read fell back')): | |
| arrow = read.to_arrow(plan.splits(), parallelism=2) | |
| assert sorted(zip( | |
| arrow.column('v').to_pylist(), | |
| arrow.column('_row_kind').to_pylist(), | |
| )) == [('0', '+I'), ('1', '-U'), ('2', '+U'), ('3', '-D')] | |
| # Initial streaming bootstrap remains a merged snapshot, where the last -D removes the key. | |
| initial = scan._create_initial_plan(table.snapshot_manager().get_snapshot_by_id(4)) | |
| assert all(not split.is_streaming for split in initial.splits()) | |
| assert list(table.new_read_builder().new_read().to_iterator(initial.splits())) == [] | |
| @pytest.mark.native_plan | |
| def test_stream_read_builder_combines_native_nested_projection_and_row_kind( | |
| catalog): | |
| from pypaimon.read.streaming_table_scan import StreamTableScan | |
| schema = pa.schema([ | |
| ('k', pa.int64()), | |
| ('payload', pa.struct([ | |
| ('score', pa.int32()), | |
| ('ignored', pa.string()), | |
| ])), | |
| ]) | |
| catalog.create_table( | |
| 'default.stream_nested', | |
| Schema.from_pyarrow_schema(schema, options={'bucket': '-1'}), | |
| False, | |
| ) | |
| table = catalog.get_table('default.stream_nested') | |
| write_builder = table.new_batch_write_builder() | |
| writer, commit = write_builder.new_write(), write_builder.new_commit() | |
| try: | |
| writer.write_arrow(pa.Table.from_pylist([ | |
| {'k': 1, 'payload': {'score': 10, 'ignored': 'a'}}, | |
| {'k': 2, 'payload': None}, | |
| ], schema=schema)) | |
| commit.commit(writer.prepare_commit()) | |
| finally: | |
| writer.close() | |
| commit.close() | |
| native_table = table.copy({ | |
| 'scan.native-plan.enabled': 'true', | |
| 'read.native.enabled': 'true', | |
| }) | |
| builder = (native_table.new_stream_read_builder() | |
| .with_projection(['payload.score', 'k']) | |
| .with_include_row_kind()) | |
| # Restore the public stream scan to consume this snapshot as a delta. | |
| scan = builder.new_streaming_scan() | |
| assert isinstance(scan, StreamTableScan) | |
| snapshot = native_table.snapshot_manager().get_latest_snapshot() | |
| scan.restore(snapshot.id) | |
| plan = scan.plan() | |
| with patch( | |
| 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', | |
| side_effect=AssertionError('streaming nested native read fell back')): | |
| rows = builder.new_read().to_arrow(plan.splits()).to_pylist() | |
| assert rows == [ | |
| {'_row_kind': '+I', 'payload_score': 10, 'k': 1}, | |
| {'_row_kind': '+I', 'payload_score': None, 'k': 2}, | |
| ] | |
| @pytest.mark.native_plan | |
| def test_streaming_changelog_frames_use_native_plan_and_read(catalog): | |
| import asyncio | |
| table = _table(catalog, 'native_changelog', True, { | |
| 'bucket': '1', | |
| 'changelog-producer': 'input', | |
| 'source.split.target-size': '1 b', | |
| 'source.split.open-file-cost': '1 b', | |
| }) | |
| _write(table, 100, [{'k': 1, 'v': 'a'}, {'k': 2, 'v': 'b'}]) | |
| _write(table, 200, [{'k': 3, 'v': 'c'}]) | |
| native_table = table.copy({ | |
| 'scan.native-plan.enabled': 'true', | |
| 'read.native.enabled': 'true', | |
| }) | |
| predicate = (native_table.new_read_builder().new_predicate_builder() | |
| .greater_than('k', 1)) | |
| builder = (native_table.new_stream_read_builder() | |
| .with_filter(predicate) | |
| .with_projection(['v']) | |
| .with_include_row_kind()) | |
| scan = builder.new_streaming_scan() | |
| scan.next_snapshot_id = 1 | |
| async def first_two_frames(): | |
| plans = [] | |
| async for plan in scan.stream(): | |
| plans.append(plan) | |
| if len(plans) == 2: | |
| return plans | |
| with patch( | |
| 'pypaimon.execution.python.read.streaming_table_scan.ManifestFileManager', | |
| side_effect=AssertionError( | |
| 'streaming changelog native plan fell back to Python')): | |
| plans = asyncio.run(first_two_frames()) | |
| assert [plan.snapshot_id for plan in plans] == [1, 2] | |
| assert all( | |
| getattr(split, '_native_split', None) is not None | |
| for plan in plans for split in plan.splits()) | |
| assert all( | |
| file.file_name.startswith('changelog-') | |
| for plan in plans for split in plan.splits() for file in split.files) | |
| with patch( | |
| 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', | |
| side_effect=AssertionError( | |
| 'streaming changelog native read fell back to Python')): | |
| rows = [ | |
| row | |
| for plan in plans | |
| for row in builder.new_read().to_arrow( | |
| plan.splits(), parallelism=2).to_pylist() | |
| ] | |
| assert rows == [ | |
| {'_row_kind': '+I', 'v': 'b'}, | |
| {'_row_kind': '+I', 'v': 'c'}, | |
| ] | |
| def test_streaming_overwrite_is_skipped_by_default_like_java(catalog): | |
| import asyncio | |
| table = _table(catalog, 'native_overwrite_changelog', True, { | |
| 'bucket': '1', | |
| 'changelog-producer': 'input', | |
| }) | |
| _write(table, 100, [{'k': 1, 'v': 'before'}]) | |
| _write(table, 200, [{'k': 2, 'v': 'after'}], overwrite=True) | |
| overwrite = table.snapshot_manager().get_latest_snapshot() | |
| assert overwrite.commit_kind == 'OVERWRITE' | |
| assert overwrite.changelog_manifest_list is None | |
| _write(table, 300, [{'k': 3, 'v': 'next'}]) | |
| builder = (table.new_stream_read_builder() | |
| .with_projection(['v']) | |
| .with_include_row_kind()) | |
| scan = builder.new_streaming_scan() | |
| scan.next_snapshot_id = overwrite.id | |
| async def next_plan(): | |
| async for plan in scan.stream(): | |
| return plan | |
| plan = asyncio.run(next_plan()) | |
| assert plan.snapshot_id == 3 | |
| assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [ | |
| {'_row_kind': '+I', 'v': 'next'}] | |
| def test_streaming_reader_honors_explicit_split_deletion_vector(catalog, native, tmp_path): | |
| from pypaimon.execution.python.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector | |
| from pypaimon.table.source.deletion_file import DeletionFile | |
| table = _table(catalog, 'explicit_dv', True, {'bucket': '1'}) | |
| _write(table, 100, [{'k': key, 'v': str(key)} for key in (1, 2, 3)]) | |
| plan, rows = _read(table, native, (0, 100)) | |
| assert len(rows) == 3 | |
| vector = BitmapDeletionVector() | |
| vector.delete(1) | |
| encoded = vector.serialize() | |
| path = tmp_path / 'explicit-dv' | |
| path.write_bytes(encoded) | |
| assert len(plan.splits()) == 1 | |
| split = plan.splits()[0] | |
| assert split.is_streaming | |
| # The planner cannot attach an endpoint DV. Build the reader input split | |
| # with that DV instead of mutating a previously planned (and cached) split. | |
| dv_split = DataSplit( | |
| files=split.files, | |
| partition=split.partition, | |
| bucket=split.bucket, | |
| raw_convertible=split.raw_convertible, | |
| data_deletion_files=[DeletionFile(str(path), 0, len(encoded) - 8, 1)], | |
| snapshot_id=split.snapshot_id, | |
| is_streaming=split.is_streaming, | |
| bucket_path=split.bucket_path, | |
| total_buckets=split.total_buckets, | |
| ) | |
| read_table = table.copy({'read.native.enabled': str(native).lower()}) | |
| with ExitStack() as stack: | |
| if native: | |
| stack.enter_context(patch( | |
| 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', | |
| side_effect=AssertionError('explicit DV native read fell back'))) | |
| result = read_table.new_read_builder().new_read().to_arrow([dv_split]).to_pylist() | |
| assert result == [{'k': 1, 'v': '1'}, {'k': 3, 'v': '3'}] | |
| @pytest.mark.parametrize('window', [(100, 100), (200, 100)]) | |
| def test_empty_table_timestamp_windows_follow_java(catalog, native, window): | |
| table = _table(catalog, 'empty_window') | |
| plan, rows = _read(table, native, window) | |
| assert rows == [] | |
| assert plan.snapshot_id is None | |
| def test_reverse_window_is_rejected_when_snapshots_exist(catalog): | |
| table = _table(catalog, 'reverse_window') | |
| _write(table, 100, [{'k': 1, 'v': 'base'}]) | |
| with pytest.raises(ValueError, match='Ending timestamp'): | |
| table.copy({'incremental-between-timestamp': '200,100'}).new_read_builder().new_scan().plan() | |
| def test_equal_window_is_empty_for_populated_tables(native, history): | |
| plan, rows = _read(history, native, (100, 100)) | |
| assert rows == [] | |
| assert plan.snapshot_id is None | |
| def test_timestamp_selection_is_deferred_until_planning(native, catalog): | |
| table = _table(catalog, 'deferred_window') | |
| builder = table.copy({ | |
| 'scan.native-plan.enabled': str(native).lower(), | |
| 'read.native.enabled': str(native).lower(), | |
| 'incremental-between-timestamp': '0,200', | |
| }).new_read_builder() | |
| scan = builder.new_scan() | |
| _write(table, 100, [{'k': 1, 'v': 'selected'}]) | |
| with ExitStack() as stack: | |
| if native: | |
| stack.enter_context(patch.object( | |
| scan, '_create_file_scanner', side_effect=AssertionError('Python scanner ran'))) | |
| plan = scan.plan() | |
| assert plan.snapshot_id == 1 | |
| assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [{'k': 1, 'v': 'selected'}] | |
| def test_timestamp_auto_reads_physical_changelog(native, catalog): | |
| table = _table(catalog, 'batch_changelog', True, {'bucket': '1', 'changelog-producer': 'input'}) | |
| _write(table, 100, [{'k': 1, 'v': 'old'}]) | |
| _write(table, 200, [{'k': 1, 'v': 'new'}, {'k': 2, 'v': 'insert'}]) | |
| _write(table, 300, [{'k': 9, 'v': 'overwrite'}], overwrite=True) | |
| plan, rows = _read(table, native, (100, 300)) | |
| assert plan.snapshot_id == 3 | |
| assert all(split.snapshot_id == 3 for split in plan.splits()) | |
| assert all(file.file_name.startswith('changelog-') for split in plan.splits() for file in split.files) | |
| assert sorted((row['k'], row['v']) for row in rows) == [(1, 'new'), (2, 'insert')] | |
| def _read_incremental_options(table, options, configure_scanner=False): | |
| builder = table.copy(options).new_read_builder() | |
| scan = builder.new_scan() | |
| if configure_scanner: | |
| assert scan.file_scanner.is_streaming | |
| assert not scan.file_scanner.skip_level0 | |
| plan = scan.plan() | |
| return plan, builder.new_read().to_arrow(plan.splits()).to_pylist() | |
| @pytest.mark.parametrize('primary_key', [False, True]) | |
| @pytest.mark.parametrize('window,expected', [ | |
| ('1,2', [2]), ('0,1', [1]), ('+1,02', [2]), ('1,2,', [2]), | |
| ('1,3', [2, 3]), ('2,2', []), ('99,99', []), | |
| ]) | |
| def test_snapshot_id_window(catalog, primary_key, window, expected): | |
| table = _table(catalog, 'snapshot_window', primary_key, {'bucket': '1'} if primary_key else None) | |
| for key in range(1, 4): | |
| _write(table, key * 100, [{'k': key, 'v': str(key)}]) | |
| plan, rows = _read_incremental_options(table, {'incremental-between': window}, configure_scanner=True) | |
| assert sorted(row['k'] for row in rows) == expected | |
| assert plan.snapshot_id == (max(expected) if expected else None) | |
| @pytest.mark.parametrize('window,message', [ | |
| ('3,1', 'Ending snapshotId should >= starting snapshotId 1. [3]'), | |
| ('-1,2', 'The specified scan snapshotId range [-1, 2] is out of available snapshotId range [1, 5].'), | |
| ('1,6', 'The specified scan snapshotId range [1, 6] is out of available snapshotId range [1, 5].'), | |
| (',2', "Tag name shouldn't be blank."), | |
| ]) | |
| def test_snapshot_id_window_validation(history, window, message): | |
| with pytest.raises(ValueError) as error: | |
| _read_incremental_options(history, {'incremental-between': window}) | |
| assert str(error.value) == message | |
| @pytest.mark.parametrize('window', [ | |
| 'missing,end', '1,end', ' 1,2', '1,2.0', '1_0,20', | |
| '1,9223372036854775808', '-9223372036854775809,2', | |
| ]) | |
| def test_snapshot_id_window_invalid_ids(catalog, window): | |
| table = _table(catalog, 'invalid_ids') | |
| with pytest.raises(ValueError) as error: | |
| _read_incremental_options(table, {'incremental-between': window}) | |
| start, end = window.split(',') | |
| assert str(error.value) == ( | |
| "Didn't find two tags for start '%s' and end '%s', and they are not two snapshot Ids. " | |
| "Please set two tags or two snapshot Ids." % (start, end)) | |
| def test_snapshot_id_window_empty_table(catalog): | |
| table = _table(catalog, 'empty_ids') | |
| assert _read_incremental_options(table, {'incremental-between': '0,99'})[1] == [] | |
| with pytest.raises(ValueError, match='Ending snapshotId'): | |
| _read_incremental_options(table, {'incremental-between': '2,1'}) | |
| def test_snapshot_id_window_expired_start(history): | |
| manager = history.snapshot_manager() | |
| history.file_io.delete_quietly(manager.get_snapshot_path(1)) | |
| assert _read_incremental_options(history, {'incremental-between': '1,2'})[0].snapshot_id == 2 | |
| with pytest.raises(ValueError, match=r'available snapshotId range \[2, 5\]'): | |
| _read_incremental_options(history, {'incremental-between': '0,2'}) | |
| def test_snapshot_id_window_earliest_beyond_default_probe(history): | |
| manager = history.snapshot_manager() | |
| for snapshot_id in (1, 2): | |
| history.file_io.delete_quietly(manager.get_snapshot_path(snapshot_id)) | |
| with patch('pypaimon.snapshot.snapshot_manager.EARLIEST_SNAPSHOT_DEFAULT_RETRY_NUM', 1): | |
| assert _read_incremental_options(history, {'incremental-between': '2,3'})[0].snapshot_id == 3 | |
| @pytest.mark.parametrize('names', [('tag1', 'tag2'), ('1', '2')]) | |
| def test_tag_window_resolves_tags_before_snapshot_ids(history, names): | |
| manager = history.snapshot_manager() | |
| for name, snapshot_id in zip(names, (2, 3)): | |
| history.tag_manager().create_tag(manager.get_snapshot_by_id(snapshot_id), name) | |
| plan, rows = _read_incremental_options(history, { | |
| 'incremental-between': ','.join(names), 'incremental-between-tag-to-snapshot': 'true'}) | |
| assert plan.snapshot_id == 3 | |
| assert sorted((row['k'], row['v']) for row in rows) == [(1, 'latest'), (4, 'fourth')] | |
| def test_tag_window_default_diff_and_ordering(history): | |
| manager = history.snapshot_manager() | |
| for name, snapshot_id in [('start', 1), ('end', 3)]: | |
| history.tag_manager().create_tag(manager.get_snapshot_by_id(snapshot_id), name) | |
| with pytest.raises(NotImplementedError, match='diff'): | |
| _read_incremental_options(history, {'incremental-between': 'start,end'}) | |
| with pytest.raises(ValueError) as error: | |
| _read_incremental_options(history, {'incremental-between': 'end,start'}) | |
| assert str(error.value) == 'Tag end start with snapshot id 1 should be >= tag start end with snapshot id 3' | |
| assert _read_incremental_options(history, {'incremental-between': 'start,start'})[1] == [] | |
| with pytest.raises(ValueError, match='Please set two tags or two snapshot Ids'): | |
| _read_incremental_options(history, {'incremental-between': 'start,3'}) | |
| def test_tag_window_uses_branch_history(catalog): | |
| table = _table(catalog, 'tag_branch') | |
| _write(table, 100, [{'k': 1, 'v': 'base'}]) | |
| table.create_tag('base') | |
| catalog.create_branch(table.identifier, 'test', tag_name='base') | |
| branch = catalog.get_table(Identifier('default', 'tag_branch', branch='test')) | |
| for current, value in ((table, 'main'), (branch, 'branch')): | |
| _write(current, 200, [{'k': 2, 'v': value}]) | |
| current.create_tag('end') | |
| _, rows = _read_incremental_options(branch, { | |
| 'incremental-between': 'base,end', 'incremental-between-tag-to-snapshot': 'true'}) | |
| assert rows == [{'k': 2, 'v': 'branch'}] | |
| @pytest.mark.parametrize('selector,window', [ | |
| ('incremental-between', '1,3'), ('incremental-between-timestamp', '100,300'), | |
| ]) | |
| @pytest.mark.parametrize('producer', ['none', 'input']) | |
| @pytest.mark.parametrize('mode', ['auto', 'delta', 'changelog', 'diff', 'invalid']) | |
| def test_incremental_scan_mode(catalog, selector, window, producer, mode): | |
| table = _table(catalog, 'scan_modes', True, {'bucket': '1', 'changelog-producer': producer}) | |
| for timestamp, value in ((100, '1'), (200, '2'), (300, '1')): | |
| _write(table, timestamp, [{'k': 1, 'v': value}]) | |
| options = {selector: window, 'incremental-between-scan-mode': mode} | |
| if mode in ('diff', 'invalid'): | |
| with pytest.raises(NotImplementedError if mode == 'diff' else ValueError, match=mode): | |
| _read_incremental_options(table, options) | |
| return | |
| plan, rows = _read_incremental_options(table, options) | |
| assert sorted(row['v'] for row in rows) == ([] if mode == 'changelog' and producer == 'none' else ['1', '2']) | |
| changelog = mode == 'changelog' or (mode == 'auto' and producer != 'none') | |
| assert all(file.file_name.startswith('changelog-') == changelog | |
| for split in plan.splits() for file in split.files) | |
| def test_incremental_scan_mode_skips_compact_and_overwrite(history): | |
| manager = history.snapshot_manager() | |
| path = manager.get_snapshot_path(3) | |
| snapshot = json.loads(history.file_io.read_file_utf8(path)) | |
| snapshot['commitKind'] = 'COMPACT' | |
| history.file_io.overwrite_file_utf8(path, json.dumps(snapshot)) | |
| plan, rows = _read_incremental_options(history, {'incremental-between': '1,5'}) | |
| assert plan.snapshot_id == 5 | |
| assert sorted((row['k'], row['v']) for row in rows) == [(1, 'intermediate'), (3, 'third'), (5, 'fifth')] | |
| @pytest.mark.parametrize('mode', ['default', 'incremental']) | |
| def test_incremental_to_auto_tag_is_explicitly_unsupported(catalog, mode): | |
| table = _table(catalog, 'auto_tag') | |
| with pytest.raises(NotImplementedError, match='incremental-to-auto-tag'): | |
| _read_incremental_options(table, {'scan.mode': mode, 'incremental-to-auto-tag': '2026-10-27'}) |