Web console: targetRowsPerSegment for hashed partionin (#10500)
* Web console: targetRowsPerSegment for hashed partionin
Added `targetRowsPerSegment` to the web console for hashed partition for both
the auto compaction view and as part of the ingestion workflow.
The help text was also updated to indicate when a user should care about
updating these fields
* code review
* update test snapshots
* oops
diff --git a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap
index 57ead47..2cbd2be 100644
--- a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap
+++ b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap
@@ -80,7 +80,26 @@
Object {
"defined": [Function],
"info": <React.Fragment>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ </React.Fragment>,
+ "label": "Target rows per segment",
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
</React.Fragment>,
"label": "Num shards",
"name": "tuningConfig.partitionsSpec.numShards",
@@ -298,7 +317,26 @@
Object {
"defined": [Function],
"info": <React.Fragment>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ </React.Fragment>,
+ "label": "Target rows per segment",
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
</React.Fragment>,
"label": "Num shards",
"name": "tuningConfig.partitionsSpec.numShards",
@@ -516,7 +554,26 @@
Object {
"defined": [Function],
"info": <React.Fragment>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ </React.Fragment>,
+ "label": "Target rows per segment",
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
</React.Fragment>,
"label": "Num shards",
"name": "tuningConfig.partitionsSpec.numShards",
@@ -734,7 +791,26 @@
Object {
"defined": [Function],
"info": <React.Fragment>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ </React.Fragment>,
+ "label": "Target rows per segment",
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
</React.Fragment>,
"label": "Num shards",
"name": "tuningConfig.partitionsSpec.numShards",
diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
index 44bee9a..d2b8055 100644
--- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
+++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
@@ -75,15 +75,43 @@
},
// partitionsSpec type: hashed
{
+ name: 'tuningConfig.partitionsSpec.targetRowsPerSegment',
+ label: 'Target rows per segment',
+ type: 'number',
+ defined: (t: CompactionConfig) =>
+ deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' &&
+ !deepGet(t, 'tuningConfig.partitionsSpec.numShards'),
+ info: (
+ <>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions,
+ consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the
+ target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ </>
+ ),
+ },
+ {
name: 'tuningConfig.partitionsSpec.numShards',
label: 'Num shards',
type: 'number',
- defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed',
+ defined: (t: CompactionConfig) =>
+ deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' &&
+ !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
info: (
<>
- Directly specify the number of shards to create. If this is specified and 'intervals' is
- specified in the granularitySpec, the index task can skip the determine intervals/partitions
- pass through the data.
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for
+ compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is
+ specified in the granularitySpec, the index task can skip the determine
+ intervals/partitions pass through the data.
+ </p>
</>
),
},
@@ -211,7 +239,12 @@
deepGet(compactionConfig, 'tuningConfig.partitionsSpec.type') || 'dynamic';
switch (partitionsSpecType) {
// case 'dynamic': // Nothing to check for dynamic
- // case 'hashed': // Nothing to check for hashed
+ case 'hashed':
+ return !(
+ Boolean(deepGet(compactionConfig, 'tuningConfig.partitionsSpec.targetRowsPerSegment')) &&
+ Boolean(deepGet(compactionConfig, 'tuningConfig.partitionsSpec.numShards'))
+ );
+ break;
case 'single_dim':
if (!deepGet(compactionConfig, 'tuningConfig.partitionsSpec.partitionDimension')) {
return false;
diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx
index 996f8e0..73f6d43 100644
--- a/web-console/src/utils/ingestion-spec.tsx
+++ b/web-console/src/utils/ingestion-spec.tsx
@@ -2114,6 +2114,11 @@
if (!intervals) return true;
switch (deepGet(tuningConfig, 'partitionsSpec.type')) {
+ case 'hashed':
+ return (
+ Boolean(deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment')) &&
+ Boolean(deepGet(tuningConfig, 'partitionsSpec.numShards'))
+ );
case 'single_dim':
if (!deepGet(tuningConfig, 'partitionsSpec.partitionDimension')) return true;
const hasTargetRowsPerSegment = Boolean(
@@ -2182,15 +2187,44 @@
},
// partitionsSpec type: hashed
{
+ name: 'partitionsSpec.targetRowsPerSegment',
+ label: 'Target rows per segment',
+ type: 'number',
+ defined: (t: TuningConfig) =>
+ deepGet(t, 'partitionsSpec.type') === 'hashed' &&
+ !deepGet(t, 'partitionsSpec.numShards'),
+ info: (
+ <>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition
+ dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to
+ the target assuming evenly distributed keys. Defaults to 5 million if numShards is
+ null.
+ </p>
+ </>
+ ),
+ },
+ {
name: 'partitionsSpec.numShards',
label: 'Num shards',
type: 'number',
- defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed',
+ defined: (t: TuningConfig) =>
+ deepGet(t, 'partitionsSpec.type') === 'hashed' &&
+ !deepGet(t, 'partitionsSpec.targetRowsPerSegment'),
info: (
<>
- Directly specify the number of shards to create. If this is specified and 'intervals'
- is specified in the granularitySpec, the index task can skip the determine
- intervals/partitions pass through the data.
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for
+ compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and
+ 'intervals' is specified in the granularitySpec, the index task can skip the
+ determine intervals/partitions pass through the data.
+ </p>
</>
),
},
