Merge pull request #15511 from pcoet/noGradle
diff --git a/.test-infra/metrics/grafana/Dockerfile b/.test-infra/metrics/grafana/Dockerfile
index ff5b66d..5eadd2a 100644
--- a/.test-infra/metrics/grafana/Dockerfile
+++ b/.test-infra/metrics/grafana/Dockerfile
@@ -16,7 +16,9 @@
# limitations under the License.
################################################################################
-FROM grafana/grafana:6.7.3
+FROM grafana/grafana:8.1.2
+
+RUN grafana-cli plugins install marcusolsson-json-datasource
COPY ./provisioning /etc/beamgrafana/provisioning
COPY ./dashboards /etc/beamgrafana/dashboards
diff --git a/.test-infra/metrics/grafana/dashboards/github_actions_post-commit_tests.json b/.test-infra/metrics/grafana/dashboards/github_actions_post-commit_tests.json
index 11ad969..d5be402 100644
--- a/.test-infra/metrics/grafana/dashboards/github_actions_post-commit_tests.json
+++ b/.test-infra/metrics/grafana/dashboards/github_actions_post-commit_tests.json
@@ -280,7 +280,7 @@
"type": "table"
},
{
- "datasource": null,
+ "datasource": "Java Tests",
"fieldConfig": {
"defaults": {
"color": {
@@ -420,7 +420,7 @@
"type": "piechart"
},
{
- "datasource": null,
+ "datasource": "Java Tests",
"fieldConfig": {
"defaults": {
"color": {
diff --git a/.test-infra/metrics/grafana/dashboards/post-commit_tests.json b/.test-infra/metrics/grafana/dashboards/post-commit_tests.json
index 0a7b2a4..2914306 100644
--- a/.test-infra/metrics/grafana/dashboards/post-commit_tests.json
+++ b/.test-infra/metrics/grafana/dashboards/post-commit_tests.json
@@ -10,6 +10,12 @@
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
@@ -17,11 +23,10 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
- "id": 1,
"links": [],
"panels": [
{
- "content": "This dashboard tracks Post-commit test reliability over-time.\n\n* [Post-commit test policies](https://beam.apache.org/contribute/postcommits-policies/)\n* [Existing test failure issues](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC)\n* [File a new test failure issue](https://s.apache.org/beam-test-failure)",
+ "datasource": null,
"gridPos": {
"h": 4,
"w": 24,
@@ -30,7 +35,11 @@
},
"id": 11,
"links": [],
- "mode": "markdown",
+ "options": {
+ "content": "This dashboard tracks Post-commit test reliability over-time.\n\n* [Post-commit test policies](https://beam.apache.org/contribute/postcommits-policies/)\n* [Existing test failure issues](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC)\n* [File a new test failure issue](https://s.apache.org/beam-test-failure)",
+ "mode": "markdown"
+ },
+ "pluginVersion": "8.1.2",
"title": "Dashboard guidelines",
"type": "text"
},
@@ -68,14 +77,62 @@
"noDataState": "keep_state",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "decimals": 0,
"description": "Percent reliability of all post-commit job runs for a given week.\n\nUnreliability of a test suite impact developer productivity by forcing contributors to re-run tests. When tests are consistently unreliable, developers will simply ignore them.\n\nWe aim for >= 70% reliability per test suite.",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "% successful runs",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "decimals": 1,
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "transparent",
+ "value": 0.7
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 7,
"w": 24,
@@ -83,31 +140,20 @@
"y": 4
},
"id": 6,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": false,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -115,7 +161,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n DATE_TRUNC('week', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (((job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')) OR job_name like '%_Cron')\nGROUP BY\n time, job_name\norder BY\n job_name, time\n",
+ "rawSql": "SELECT\n DATE_TRUNC('week', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (((job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')) OR job_name like '%_Cron')\nGROUP BY\n time, job_name\norder BY\n time, job_name\n",
"refId": "A",
"select": [
[
@@ -137,55 +183,10 @@
]
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "lt",
- "value": 0.7
- }
- ],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Post-commit reliability per week",
- "tooltip": {
- "shared": true,
- "sort": 1,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 1,
- "format": "percentunit",
- "label": "% successful runs",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -221,14 +222,62 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "decimals": 0,
"description": "Percent reliability of all post-commit job runs per-day.\n\nUnreliability of a test suite impact developer productivity by forcing contributors to re-run tests. When tests are consistently unreliable, developers will simply ignore them.\n\nWe aim for >= 70% reliability per test suite.",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "% successful runs",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "decimals": 1,
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "transparent",
+ "value": 0.7
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 12,
"w": 15,
@@ -236,31 +285,20 @@
"y": 11
},
"id": 9,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sideWidth": null,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "hidden",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -268,7 +306,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n DATE_TRUNC('day', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n job_name, time\n",
+ "rawSql": "SELECT\n DATE_TRUNC('day', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n time, job_name\n",
"refId": "A",
"select": [
[
@@ -290,61 +328,197 @@
]
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "lt",
- "value": 0.7
- }
- ],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Post-commit reliability per day",
- "tooltip": {
- "shared": true,
- "sort": 1,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 1,
- "format": "percentunit",
- "label": "% successful runs",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "columns": [],
"datasource": "BeamPSQL",
"description": "List of jobs which have failed. Click on the job to view it in Jenkins.",
- "fontSize": "100%",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "build_url"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Build Url"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "links",
+ "value": [
+ {
+ "targetBlank": true,
+ "title": "Link to Jenkins job.",
+ "url": "${__cell:raw}"
+ }
+ ]
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "job_name"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Job Name"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "links",
+ "value": [
+ {
+ "targetBlank": true,
+ "title": "View Jenkins job: ${__cell_1}_${__cell_2}",
+ "url": "${__cell_0:raw}"
+ }
+ ]
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "build_id"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Build ID"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "links",
+ "value": [
+ {
+ "targetBlank": true,
+ "title": "View Jenkins job: ${__cell_1}_${__cell_2}",
+ "url": "${__cell_0:raw}"
+ }
+ ]
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "build_timestamp"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Start Time"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "unit",
+ "value": "time: MM/DD/YY h:mm:ss a"
+ },
+ {
+ "id": "links",
+ "value": [
+ {
+ "targetBlank": true,
+ "title": "View Jenkins job: ${__cell_1}_${__cell_2}",
+ "url": "${__cell_0:raw}"
+ }
+ ]
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ }
+ ]
+ },
"gridPos": {
"h": 12,
"w": 9,
@@ -355,109 +529,15 @@
"id": 8,
"links": [
{
- "includeVars": false,
"targetBlank": true,
"title": "Beam Jenkins",
- "type": "absolute",
"url": "https://ci-beam.apache.org/"
}
],
- "pageSize": null,
- "scroll": true,
- "showHeader": true,
- "sort": {
- "col": 0,
- "desc": true
+ "options": {
+ "showHeader": true
},
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "link": false,
- "pattern": "Time",
- "type": "date"
- },
- {
- "alias": "Build Url",
- "colorMode": null,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": true,
- "linkTargetBlank": true,
- "linkTooltip": "Link to Jenkins job.",
- "linkUrl": "${__cell:raw}",
- "mappingType": 1,
- "pattern": "build_url",
- "thresholds": [],
- "type": "hidden",
- "unit": "short"
- },
- {
- "alias": "Job Name",
- "colorMode": null,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": true,
- "linkTargetBlank": true,
- "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}",
- "linkUrl": "${__cell_0:raw}",
- "mappingType": 1,
- "pattern": "job_name",
- "thresholds": [],
- "type": "string",
- "unit": "short"
- },
- {
- "alias": "Build ID",
- "colorMode": null,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 0,
- "link": true,
- "linkTargetBlank": true,
- "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}",
- "linkUrl": "${__cell_0:raw}",
- "mappingType": 1,
- "pattern": "build_id",
- "thresholds": [],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "Start Time",
- "colorMode": null,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "dateFormat": "MM/DD/YY h:mm:ss a",
- "decimals": 2,
- "link": true,
- "linkTargetBlank": true,
- "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}",
- "linkUrl": "${__cell_0:raw}",
- "mappingType": 1,
- "pattern": "build_timestamp",
- "thresholds": [],
- "type": "date",
- "unit": "short"
- }
- ],
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -489,18 +569,71 @@
],
"timeShift": null,
"title": "Failed builds",
- "transform": "table",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ }
+ ],
"type": "table"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "decimals": 1,
"description": "Execution time for each post-commit job",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "Average job duration",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 15,
@@ -508,29 +641,20 @@
"y": 23
},
"id": 5,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "hidden",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -538,7 +662,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n build_timestamp as time,\n build_duration as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name LIKE '%_PR')\nORDER BY\n job_name, time",
+ "rawSql": "SELECT\n build_timestamp as time,\n build_duration as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name LIKE '%_PR')\nORDER BY\n time, job_name",
"refId": "A",
"select": [
[
@@ -560,57 +684,98 @@
]
}
],
- "thresholds": [],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Post-commit job duration",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "ms",
- "label": "Average job duration",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "decimals": 0,
"description": "Tracks the count of test failure JIRA issues currently open.",
- "fill": 3,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "# of JIRA issues",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_open"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#eab839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "currently_failing"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#bf1b00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"gridPos": {
"h": 8,
"w": 9,
@@ -618,45 +783,24 @@
"y": 23
},
"id": 14,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 1,
"links": [
{
"targetBlank": true,
"title": "Jira tickets",
- "type": "absolute",
"url": "https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC"
}
],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "total_open",
- "color": "#eab839"
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
},
- {
- "alias": "currently_failing",
- "color": "#bf1b00"
+ "tooltip": {
+ "mode": "single"
}
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"format": "time_series",
@@ -713,51 +857,14 @@
]
}
],
- "thresholds": [],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Test Failure JIRA issues",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "label": "# of JIRA issues",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
}
],
"refresh": false,
- "schemaVersion": 18,
+ "schemaVersion": 30,
"style": "dark",
"tags": [],
"templating": {
@@ -793,5 +900,5 @@
"timezone": "",
"title": "Post-commit Test Reliability",
"uid": "D81lW0pmk",
- "version": 46
+ "version": 2
}
diff --git a/.test-infra/metrics/grafana/dashboards/pre-commit_tests.json b/.test-infra/metrics/grafana/dashboards/pre-commit_tests.json
index e5ab46e..a518b2a 100644
--- a/.test-infra/metrics/grafana/dashboards/pre-commit_tests.json
+++ b/.test-infra/metrics/grafana/dashboards/pre-commit_tests.json
@@ -8,6 +8,12 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
@@ -15,7 +21,6 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
- "id": 2,
"links": [],
"panels": [
{
@@ -52,13 +57,60 @@
"noDataState": "keep_state",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
"description": "Execution time for each pre-commit job.\n\nLong test suite execution impacts developer productivity by delaying the quality signal of a pull request of current HEAD. If tests are consistently slow, developers won't wait for them to complete.\n\nWe aim for under 2 hour execution per test suite, but ideally under 30 mins.",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "Average job duration",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 7200000
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 24,
@@ -66,31 +118,20 @@
"y": 0
},
"id": 4,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -98,7 +139,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n build_timestamp as time,\n build_duration as value,\n substring(job_name from 'beam_PreCommit_#\"%#\"_(Cron|Commit)' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND build_result = 'SUCCESS'\n AND ((job_name LIKE 'beam_PreCommit_%_Commit')\n OR (job_name LIKE 'beam_PreCommit_%_Cron'))\nORDER BY\n metric, time",
+ "rawSql": "SELECT\n build_timestamp as time,\n build_duration as value,\n substring(job_name from 'beam_PreCommit_#\"%#\"_(Cron|Commit)' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND build_result = 'SUCCESS'\n AND ((job_name LIKE 'beam_PreCommit_%_Commit')\n OR (job_name LIKE 'beam_PreCommit_%_Cron'))\nORDER BY\n time, metric",
"refId": "A",
"select": [
[
@@ -120,62 +161,64 @@
]
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 7200000
- }
- ],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Pre-commit job duration",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "label": "Average job duration",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "dtdurationms"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 24,
@@ -183,38 +226,25 @@
"y": 8
},
"id": 6,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"format": "time_series",
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n build_timestamp as time,\n timing_queuingDurationMillis as value,\n substring(job_name from 'beam_PreCommit_#\"%#\"_(Cron|Commit|Phrase)' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND build_result = 'SUCCESS'\n AND ((job_name LIKE 'beam_PreCommit_%_Commit')\n OR (job_name LIKE 'beam_PreCommit_%_Cron')\n OR (job_name LIKE 'beam_PreCommit_%_Phrase'))\nORDER BY\n metric, time",
+ "rawSql": "SELECT\n build_timestamp as time,\n timing_queuingDurationMillis as value,\n substring(job_name from 'beam_PreCommit_#\"%#\"_(Cron|Commit|Phrase)' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND build_result = 'SUCCESS'\n AND ((job_name LIKE 'beam_PreCommit_%_Commit')\n OR (job_name LIKE 'beam_PreCommit_%_Cron')\n OR (job_name LIKE 'beam_PreCommit_%_Phrase'))\nORDER BY\n time, metric",
"refId": "A",
"select": [
[
@@ -236,54 +266,64 @@
]
}
],
- "thresholds": [],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Time in queue",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "dtdurationms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "dtdurationms"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 24,
@@ -291,31 +331,18 @@
"y": 16
},
"id": 8,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"aggregation": "Last",
@@ -351,49 +378,13 @@
]
}
],
- "thresholds": [],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Time in queue: 0.9 percentile on month period",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "dtdurationms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
}
],
- "schemaVersion": 18,
+ "schemaVersion": 30,
"style": "dark",
"tags": [],
"templating": {
@@ -431,5 +422,5 @@
"timezone": "utc",
"title": "Pre-commit Test Latency",
"uid": "_TNndF2iz",
- "version": 18
+ "version": 1
}
diff --git a/.test-infra/metrics/grafana/dashboards/stability_critical_jobs_status.json b/.test-infra/metrics/grafana/dashboards/stability_critical_jobs_status.json
index 83695dc..f0ffe63 100644
--- a/.test-infra/metrics/grafana/dashboards/stability_critical_jobs_status.json
+++ b/.test-infra/metrics/grafana/dashboards/stability_critical_jobs_status.json
@@ -8,6 +8,12 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
@@ -15,11 +21,10 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
- "id": 3,
"links": [],
"panels": [
{
- "content": "The graph shows: average greenness of critical post-commit tests jobs per week. This graph show health of our project.\n\nTable shows list of relevant jobs failures during selected time interval (You can change time interval on top-right corner of the dashboard). Please, triage failed jobs and update or create corresponding jira tickets. You can utilized provided links to help with this.",
+ "datasource": null,
"gridPos": {
"h": 3,
"w": 10,
@@ -28,15 +33,97 @@
},
"id": 8,
"links": [],
- "mode": "markdown",
- "options": {},
+ "options": {
+ "content": "The graph shows: average greenness of critical post-commit tests jobs per week. This graph show health of our project.\n\nTable shows list of relevant jobs failures during selected time interval (You can change time interval on top-right corner of the dashboard). Please, triage failed jobs and update or create corresponding jira tickets. You can utilized provided links to help with this.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "8.1.2",
"title": "Dashboard guidelines",
"type": "text"
},
{
- "columns": [],
"datasource": "BeamPSQL",
- "fontSize": "100%",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "build_url"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Build Url"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "links",
+ "value": [
+ {
+ "targetBlank": true,
+ "title": "Link to Jenkins job.",
+ "url": "${__cell:raw}"
+ }
+ ]
+ },
+ {
+ "id": "custom.align",
+ "value": null
+ }
+ ]
+ }
+ ]
+ },
"gridPos": {
"h": 6,
"w": 14,
@@ -46,43 +133,10 @@
"hideTimeOverride": false,
"id": 4,
"links": [],
- "options": {},
- "pageSize": null,
- "scroll": true,
- "showHeader": true,
- "sort": {
- "col": 0,
- "desc": true
+ "options": {
+ "showHeader": true
},
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "link": false,
- "pattern": "Time",
- "type": "date"
- },
- {
- "alias": "Build Url",
- "colorMode": null,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": true,
- "linkTargetBlank": true,
- "linkTooltip": "Link to Jenkins job.",
- "linkUrl": "${__cell:raw}",
- "mappingType": 1,
- "pattern": "build_url",
- "thresholds": [],
- "type": "number",
- "unit": "short"
- }
- ],
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -114,11 +168,18 @@
],
"timeShift": null,
"title": "Failed builds",
- "transform": "table",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ }
+ ],
"type": "table"
},
{
- "content": "[List existing jira tickets](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC)\n\n[Create new Jira ticket](https://issues.apache.org/jira/secure/CreateIssueDetails!init.jspa?pid=12319527&issuetype=1&summary=%5BjobName%5D%5BTestName%5D%5BIsFlake%5D%20Failure%20summary&priority=3&components=12334203&description=%3CFailure%20summary%3E%0AFailing%20job%20url:%0AJob%20history%20url:%0ARelevant%20log:)",
+ "datasource": null,
"gridPos": {
"h": 3,
"w": 10,
@@ -127,19 +188,70 @@
},
"id": 6,
"links": [],
- "mode": "markdown",
- "options": {},
+ "options": {
+ "content": "[List existing jira tickets](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC)\n\n[Create new Jira ticket](https://issues.apache.org/jira/secure/CreateIssueDetails!init.jspa?pid=12319527&issuetype=1&summary=%5BjobName%5D%5BTestName%5D%5BIsFlake%5D%20Failure%20summary&priority=3&components=12334203&description=%3CFailure%20summary%3E%0AFailing%20job%20url:%0AJob%20history%20url:%0ARelevant%20log:)",
+ "mode": "markdown"
+ },
+ "pluginVersion": "8.1.2",
"title": "Useful links",
"type": "text"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
"description": "Each data point shows aggregation for corresponding week.\nLatest (rightmost) data point aggregates all data available for current week, so it may change based on new data and should not be considered a final value.",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#3f6833",
+ "value": null
+ },
+ {
+ "color": "transparent",
+ "value": 0.7
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 7,
"w": 10,
@@ -147,29 +259,18 @@
"y": 6
},
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "options": {},
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -177,7 +278,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n DATE_TRUNC('week', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n /*\n We perform a union here to create a fake \"Python_All\" job_name in\n order to graph a new line for all the python results combined.\n */\n ( SELECT build_timestamp, build_result, job_name\n FROM jenkins_builds\n UNION\n SELECT build_timestamp, build_result, 'beam_PostCommit_Python_All' as job_name\n FROM jenkins_builds\n WHERE \n ((job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+'))\n AND NOT (job_name like '%_PR')\n ) AS critical_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND ((job_name = 'beam_PostCommit_Java') \n OR (job_name = 'beam_PostCommit_Go') \n OR (job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+')\n OR (job_name = 'beam_PostCommit_Python_Verify')\n OR (job_name = 'beam_PostCommit_Python_All')\n OR (job_name = 'beam_PostCommit_Website_Publish'))\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n job_name, time",
+ "rawSql": "SELECT\n DATE_TRUNC('week', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n /*\n We perform a union here to create a fake \"Python_All\" job_name in\n order to graph a new line for all the python results combined.\n */\n ( SELECT build_timestamp, build_result, job_name\n FROM jenkins_builds\n UNION\n SELECT build_timestamp, build_result, 'beam_PostCommit_Python_All' as job_name\n FROM jenkins_builds\n WHERE \n ((job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+'))\n AND NOT (job_name like '%_PR')\n ) AS critical_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND ((job_name = 'beam_PostCommit_Java') \n OR (job_name = 'beam_PostCommit_Go') \n OR (job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+')\n OR (job_name = 'beam_PostCommit_Python_Verify')\n OR (job_name = 'beam_PostCommit_Python_All')\n OR (job_name = 'beam_PostCommit_Website_Publish'))\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n time, job_name",
"refId": "A",
"select": [
[
@@ -199,66 +300,67 @@
]
}
],
- "thresholds": [
- {
- "colorMode": "custom",
- "fill": false,
- "line": true,
- "lineColor": "#3f6833",
- "op": "lt",
- "value": 0.7,
- "yaxis": "left"
- }
- ],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Greenness per Week (in %)",
- "tooltip": {
- "shared": true,
- "sort": 1,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "BeamPSQL",
"description": "Each data point shows aggregation for corresponding month.\nLatest (rightmost) data point aggregates all data available for current month, so it may change based on new data and should not be considered a final value.",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#3f6833",
+ "value": null
+ },
+ {
+ "color": "transparent",
+ "value": 0.7
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 7,
"w": 14,
@@ -266,30 +368,18 @@
"y": 6
},
"id": 10,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "options": {},
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.2",
"targets": [
{
"alias": "",
@@ -297,7 +387,7 @@
"group": [],
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "SELECT\n DATE_TRUNC('month', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n /*\n We perform a union here to create a fake \"Python_All\" job_name in\n order to graph a new line for all the python results combined.\n */\n ( SELECT build_timestamp, build_result, job_name\n FROM jenkins_builds\n UNION\n SELECT build_timestamp, build_result, 'beam_PostCommit_Python_All' as job_name\n FROM jenkins_builds\n WHERE \n ((job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+'))\n AND NOT (job_name like '%_PR')\n ) AS critical_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND ((job_name = 'beam_PostCommit_Java') \n OR (job_name = 'beam_PostCommit_Go') \n OR (job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+')\n OR (job_name = 'beam_PostCommit_Python_Verify')\n OR (job_name = 'beam_PostCommit_Python_All')\n OR (job_name = 'beam_PostCommit_Website_Publish'))\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n job_name, time",
+ "rawSql": "SELECT\n DATE_TRUNC('month', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n /*\n We perform a union here to create a fake \"Python_All\" job_name in\n order to graph a new line for all the python results combined.\n */\n ( SELECT build_timestamp, build_result, job_name\n FROM jenkins_builds\n UNION\n SELECT build_timestamp, build_result, 'beam_PostCommit_Python_All' as job_name\n FROM jenkins_builds\n WHERE \n ((job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+'))\n AND NOT (job_name like '%_PR')\n ) AS critical_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND ((job_name = 'beam_PostCommit_Java') \n OR (job_name = 'beam_PostCommit_Go') \n OR (job_name SIMILAR TO 'beam_PostCommit_Python[0-9]+')\n OR (job_name = 'beam_PostCommit_Python_Verify')\n OR (job_name = 'beam_PostCommit_Python_All')\n OR (job_name = 'beam_PostCommit_Website_Publish'))\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n time, job_name",
"refId": "A",
"select": [
[
@@ -319,61 +409,14 @@
]
}
],
- "thresholds": [
- {
- "colorMode": "custom",
- "fill": false,
- "line": true,
- "lineColor": "#3f6833",
- "op": "lt",
- "value": 0.7,
- "yaxis": "left"
- }
- ],
"timeFrom": null,
- "timeRegions": [],
"timeShift": null,
"title": "Greenness per Month (in %)",
- "tooltip": {
- "shared": true,
- "sort": 1,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
}
],
"refresh": false,
- "schemaVersion": 18,
+ "schemaVersion": 30,
"style": "dark",
"tags": [],
"templating": {
@@ -411,5 +454,5 @@
"timezone": "utc",
"title": "Stability critical jobs status",
"uid": "McTAiu0ik",
- "version": 1
-}
\ No newline at end of file
+ "version": 2
+}
diff --git a/.test-infra/metrics/grafana/provisioning/datasources/beamgithubjavatests-api.yaml b/.test-infra/metrics/grafana/provisioning/datasources/beamgithubjavatests-api.yaml
new file mode 100644
index 0000000..a3a0e9b
--- /dev/null
+++ b/.test-infra/metrics/grafana/provisioning/datasources/beamgithubjavatests-api.yaml
@@ -0,0 +1,34 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+apiVersion: 1
+
+deleteDatasources:
+
+datasources:
+ - name: Java Tests
+ type: marcusolsson-json-datasource
+ access: proxy
+ orgId: 1
+ url: https://api.github.com/repos/apache/beam/actions/workflows/java_tests.yml/runs
+ jsonData:
+ httpHeaderName1: "accept"
+ customQueryParameters: "per_page=100"
+ secureJsonData:
+ httpHeaderValue1: "application/vnd.github.v3+json"
+ editable: false
diff --git a/.test-infra/metrics/grafana/provisioning/datasources/beamgithubpythontests-api.yaml b/.test-infra/metrics/grafana/provisioning/datasources/beamgithubpythontests-api.yaml
new file mode 100644
index 0000000..abcd060
--- /dev/null
+++ b/.test-infra/metrics/grafana/provisioning/datasources/beamgithubpythontests-api.yaml
@@ -0,0 +1,34 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+apiVersion: 1
+
+deleteDatasources:
+
+datasources:
+ - name: Python Tests
+ type: marcusolsson-json-datasource
+ access: proxy
+ orgId: 1
+ url: https://api.github.com/repos/apache/beam/actions/workflows/python_tests.yml/runs
+ jsonData:
+ httpHeaderName1: "accept"
+ customQueryParameters: "per_page=100"
+ secureJsonData:
+ httpHeaderValue1: "application/vnd.github.v3+json"
+ editable: false
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GcpResourceIdentifiers.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GcpResourceIdentifiers.java
index 3133cc1..4c388bf 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GcpResourceIdentifiers.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GcpResourceIdentifiers.java
@@ -43,6 +43,10 @@
projectId, instanceId, tableId);
}
+ public static String cloudStorageBucket(String bucketId) {
+ return String.format("//storage.googleapis.com/buckets/%s", bucketId);
+ }
+
public static String datastoreResource(String projectId, String namespace) {
return String.format(
"//bigtable.googleapis.com/projects/%s/namespaces/%s", projectId, namespace);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java
index 57ee58a..c792719 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java
@@ -83,6 +83,8 @@
public static final String BIGTABLE_PROJECT_ID = "BIGTABLE_PROJECT_ID";
public static final String INSTANCE_ID = "INSTANCE_ID";
public static final String TABLE_ID = "TABLE_ID";
+ public static final String GCS_BUCKET = "GCS_BUCKET";
+ public static final String GCS_PROJECT_ID = "GCS_PROJECT_ID";
static {
// Note: One benefit of defining these strings above, instead of pulling them in from
@@ -116,6 +118,8 @@
BIGTABLE_PROJECT_ID.equals(extractLabel(MonitoringInfoLabels.BIGTABLE_PROJECT_ID)));
checkArgument(INSTANCE_ID.equals(extractLabel(MonitoringInfoLabels.INSTANCE_ID)));
checkArgument(TABLE_ID.equals(extractLabel(MonitoringInfoLabels.TABLE_ID)));
+ checkArgument(GCS_BUCKET.equals(extractLabel(MonitoringInfoLabels.GCS_BUCKET)));
+ checkArgument(GCS_PROJECT_ID.equals(extractLabel(MonitoringInfoLabels.GCS_PROJECT_ID)));
}
}
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index b8ec9bb..be0ade9 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -45,8 +45,8 @@
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
'dataflow.legacy_environment_major_version' : '8',
'dataflow.fnapi_environment_major_version' : '8',
- 'dataflow.legacy_container_version' : 'beam-master-20210525',
- 'dataflow.fnapi_container_version' : 'beam-master-20210524',
+ 'dataflow.legacy_container_version' : 'beam-master-20210913',
+ 'dataflow.fnapi_container_version' : 'beam-master-20210913',
'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3',
]
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java
index 538ddaa..f6ad78f 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java
@@ -58,6 +58,7 @@
import java.nio.file.FileAlreadyExistsException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -72,6 +73,9 @@
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.beam.runners.core.metrics.GcpResourceIdentifiers;
+import org.apache.beam.runners.core.metrics.MonitoringInfoConstants;
+import org.apache.beam.runners.core.metrics.ServiceCallMetric;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath;
import org.apache.beam.sdk.io.fs.MoveOptions;
@@ -454,8 +458,31 @@
@VisibleForTesting
SeekableByteChannel open(GcsPath path, GoogleCloudStorageReadOptions readOptions)
throws IOException {
- return googleCloudStorage.open(
- new StorageResourceId(path.getBucket(), path.getObject()), readOptions);
+ HashMap<String, String> baseLabels = new HashMap<>();
+ baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
+ baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
+ baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsGet");
+ baseLabels.put(
+ MonitoringInfoConstants.Labels.RESOURCE,
+ GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
+ baseLabels.put(
+ MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
+ baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
+
+ ServiceCallMetric serviceCallMetric =
+ new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
+ try {
+ SeekableByteChannel channel =
+ googleCloudStorage.open(
+ new StorageResourceId(path.getBucket(), path.getObject()), readOptions);
+ serviceCallMetric.call("ok");
+ return channel;
+ } catch (IOException e) {
+ if (e.getCause() instanceof GoogleJsonResponseException) {
+ serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
+ }
+ throw e;
+ }
}
/**
@@ -487,9 +514,35 @@
GoogleCloudStorage gcpStorage =
new GoogleCloudStorageImpl(
newGoogleCloudStorageOptions, this.storageClient, this.credentials);
- return gcpStorage.create(
- new StorageResourceId(path.getBucket(), path.getObject()),
- CreateObjectOptions.builder().setOverwriteExisting(true).setContentType(type).build());
+ HashMap<String, String> baseLabels = new HashMap<>();
+ baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
+ baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
+ baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsInsert");
+ baseLabels.put(
+ MonitoringInfoConstants.Labels.RESOURCE,
+ GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
+ baseLabels.put(
+ MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
+ baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
+
+ ServiceCallMetric serviceCallMetric =
+ new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
+ try {
+ WritableByteChannel channel =
+ gcpStorage.create(
+ new StorageResourceId(path.getBucket(), path.getObject()),
+ CreateObjectOptions.builder()
+ .setOverwriteExisting(true)
+ .setContentType(type)
+ .build());
+ serviceCallMetric.call("ok");
+ return channel;
+ } catch (IOException e) {
+ if (e.getCause() instanceof GoogleJsonResponseException) {
+ serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
+ }
+ throw e;
+ }
}
/** Returns whether the GCS bucket exists and is accessible. */
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java
index 0c4df7a..7856ddd 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java
@@ -58,7 +58,11 @@
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.RewriteResponse;
import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.hadoop.gcsio.CreateObjectOptions;
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorage;
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions;
import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions;
+import com.google.cloud.hadoop.gcsio.StorageResourceId;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -71,6 +75,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
@@ -78,6 +83,10 @@
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
+import org.apache.beam.runners.core.metrics.GcpResourceIdentifiers;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
+import org.apache.beam.runners.core.metrics.MonitoringInfoConstants;
+import org.apache.beam.runners.core.metrics.MonitoringInfoMetricName;
import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.extensions.gcp.util.GcsUtil.BatchInterface;
@@ -85,10 +94,12 @@
import org.apache.beam.sdk.extensions.gcp.util.GcsUtil.StorageObjectOrIOException;
import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath;
import org.apache.beam.sdk.io.fs.MoveOptions.StandardMoveOptions;
+import org.apache.beam.sdk.metrics.MetricsEnvironment;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.FluentBackoff;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -101,6 +112,13 @@
public class GcsUtilTest {
@Rule public ExpectedException thrown = ExpectedException.none();
+ @Before
+ public void setUp() {
+ // Setup the ProcessWideContainer for testing metrics are set.
+ MetricsContainerImpl container = new MetricsContainerImpl(null);
+ MetricsEnvironment.setProcessWideContainer(container);
+ }
+
private static GcsOptions gcsOptionsWithTestCredential() {
GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
pipelineOptions.setGcpCredential(new TestCredential());
@@ -785,6 +803,68 @@
channel.close();
}
+ @Test
+ public void testGCSReadMetricsIsSet() {
+ GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
+ GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
+ gcsUtil.setCloudStorageImpl(
+ GoogleCloudStorageOptions.builder()
+ .setAppName("Beam")
+ .setGrpcEnabled(true)
+ .setProjectId("my_project")
+ .build());
+ GoogleCloudStorageReadOptions readOptions =
+ GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();
+ assertThrows(
+ IOException.class,
+ () -> gcsUtil.open(GcsPath.fromComponents("testbucket", "testbucket"), readOptions));
+ verifyMetricWasSet("my_project", "testbucket", "GcsGet", "permission_denied", 1);
+ }
+
+ @Test
+ public void testGCSWriteMetricsIsSet() throws IOException {
+ GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
+ GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
+ GoogleCloudStorage mockStorage = Mockito.mock(GoogleCloudStorage.class);
+ gcsUtil.setCloudStorageImpl(
+ GoogleCloudStorageOptions.builder()
+ .setAppName("Beam")
+ .setGrpcEnabled(true)
+ .setProjectId("my_project")
+ .build());
+ when(mockStorage.create(
+ new StorageResourceId("testbucket", "testobject"),
+ CreateObjectOptions.builder()
+ .setOverwriteExisting(true)
+ .setContentType("type")
+ .build()))
+ .thenThrow(IOException.class);
+ GcsPath gcsPath = GcsPath.fromComponents("testbucket", "testobject");
+ assertThrows(IOException.class, () -> gcsUtil.create(gcsPath, ""));
+ verifyMetricWasSet("my_project", "testbucket", "GcsInsert", "permission_denied", 1);
+ }
+
+ private void verifyMetricWasSet(
+ String projectId, String bucketId, String method, String status, long count) {
+ // Verify the metric as reported.
+ HashMap<String, String> labels = new HashMap<>();
+ labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
+ labels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
+ labels.put(MonitoringInfoConstants.Labels.METHOD, method);
+ labels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, projectId);
+ labels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, bucketId);
+ labels.put(
+ MonitoringInfoConstants.Labels.RESOURCE,
+ GcpResourceIdentifiers.cloudStorageBucket(bucketId));
+ labels.put(MonitoringInfoConstants.Labels.STATUS, status);
+
+ MonitoringInfoMetricName name =
+ MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
+ MetricsContainerImpl container =
+ (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
+ assertEquals(count, (long) container.getCounter(name).getCumulative());
+ }
+
/** Builds a fake GoogleJsonResponseException for testing API error handling. */
private static GoogleJsonResponseException googleJsonResponseException(
final int status, final String reason, final String message) throws IOException {
diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
index 5c2f1f5..69d5d19 100644
--- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
+++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
@@ -18,9 +18,11 @@
package org.apache.beam.sdk.io.aws.options;
import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
@@ -66,6 +68,7 @@
private static final String AWS_ACCESS_KEY_ID = "awsAccessKeyId";
private static final String AWS_SECRET_KEY = "awsSecretKey";
+ private static final String SESSION_TOKEN = "sessionToken";
private static final String CREDENTIALS_FILE_PATH = "credentialsFilePath";
public static final String CLIENT_EXECUTION_TIMEOUT = "clientExecutionTimeout";
public static final String CONNECTION_MAX_IDLE_TIME = "connectionMaxIdleTime";
@@ -119,8 +122,17 @@
}
if (typeName.equals(AWSStaticCredentialsProvider.class.getSimpleName())) {
- return new AWSStaticCredentialsProvider(
- new BasicAWSCredentials(asMap.get(AWS_ACCESS_KEY_ID), asMap.get(AWS_SECRET_KEY)));
+ boolean isSession = asMap.containsKey(SESSION_TOKEN);
+ if (isSession) {
+ return new AWSStaticCredentialsProvider(
+ new BasicSessionCredentials(
+ asMap.get(AWS_ACCESS_KEY_ID),
+ asMap.get(AWS_SECRET_KEY),
+ asMap.get(SESSION_TOKEN)));
+ } else {
+ return new AWSStaticCredentialsProvider(
+ new BasicAWSCredentials(asMap.get(AWS_ACCESS_KEY_ID), asMap.get(AWS_SECRET_KEY)));
+ }
} else if (typeName.equals(PropertiesFileCredentialsProvider.class.getSimpleName())) {
return new PropertiesFileCredentialsProvider(asMap.get(CREDENTIALS_FILE_PATH));
} else if (typeName.equals(
@@ -179,11 +191,16 @@
typeSerializer.writeTypePrefixForObject(credentialsProvider, jsonGenerator);
if (credentialsProvider.getClass().equals(AWSStaticCredentialsProvider.class)) {
- jsonGenerator.writeStringField(
- AWS_ACCESS_KEY_ID, credentialsProvider.getCredentials().getAWSAccessKeyId());
- jsonGenerator.writeStringField(
- AWS_SECRET_KEY, credentialsProvider.getCredentials().getAWSSecretKey());
-
+ AWSCredentials credentials = credentialsProvider.getCredentials();
+ if (credentials.getClass().equals(BasicSessionCredentials.class)) {
+ BasicSessionCredentials sessionCredentials = (BasicSessionCredentials) credentials;
+ jsonGenerator.writeStringField(AWS_ACCESS_KEY_ID, sessionCredentials.getAWSAccessKeyId());
+ jsonGenerator.writeStringField(AWS_SECRET_KEY, sessionCredentials.getAWSSecretKey());
+ jsonGenerator.writeStringField(SESSION_TOKEN, sessionCredentials.getSessionToken());
+ } else {
+ jsonGenerator.writeStringField(AWS_ACCESS_KEY_ID, credentials.getAWSAccessKeyId());
+ jsonGenerator.writeStringField(AWS_SECRET_KEY, credentials.getAWSSecretKey());
+ }
} else if (credentialsProvider.getClass().equals(PropertiesFileCredentialsProvider.class)) {
try {
PropertiesFileCredentialsProvider specificProvider =
diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
index 9651803..0e318c9 100644
--- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
+++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
@@ -25,6 +25,7 @@
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
@@ -78,6 +79,20 @@
assertEquals(
credentialsProvider.getCredentials().getAWSSecretKey(),
deserializedCredentialsProvider.getCredentials().getAWSSecretKey());
+
+ String sessionToken = "session-token";
+ BasicSessionCredentials sessionCredentials =
+ new BasicSessionCredentials(awsKeyId, awsSecretKey, sessionToken);
+ credentialsProvider = new AWSStaticCredentialsProvider(sessionCredentials);
+ serializedCredentialsProvider = objectMapper.writeValueAsString(credentialsProvider);
+ deserializedCredentialsProvider =
+ objectMapper.readValue(serializedCredentialsProvider, AWSCredentialsProvider.class);
+ BasicSessionCredentials deserializedCredentials =
+ (BasicSessionCredentials) deserializedCredentialsProvider.getCredentials();
+ assertEquals(credentialsProvider.getClass(), deserializedCredentialsProvider.getClass());
+ assertEquals(deserializedCredentials.getAWSAccessKeyId(), awsKeyId);
+ assertEquals(deserializedCredentials.getAWSSecretKey(), awsSecretKey);
+ assertEquals(deserializedCredentials.getSessionToken(), sessionToken);
}
@Test
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
index 5051438..7453d76 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
@@ -41,7 +41,9 @@
import org.apache.beam.sdk.io.aws2.s3.SSECustomerKey;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
@@ -64,6 +66,7 @@
public class AwsModule extends SimpleModule {
private static final String ACCESS_KEY_ID = "accessKeyId";
private static final String SECRET_ACCESS_KEY = "secretAccessKey";
+ private static final String SESSION_TOKEN = "sessionToken";
public static final String CONNECTION_ACQUIRE_TIMEOUT = "connectionAcquisitionTimeout";
public static final String CONNECTION_MAX_IDLE_TIMEOUT = "connectionMaxIdleTime";
public static final String CONNECTION_TIMEOUT = "connectionTimeout";
@@ -107,10 +110,18 @@
throw new IOException(
String.format("AWS credentials provider type name key '%s' not found", typeNameKey));
}
-
if (typeName.equals(StaticCredentialsProvider.class.getSimpleName())) {
- return StaticCredentialsProvider.create(
- AwsBasicCredentials.create(asMap.get(ACCESS_KEY_ID), asMap.get(SECRET_ACCESS_KEY)));
+ boolean isSession = asMap.containsKey(SESSION_TOKEN);
+ if (isSession) {
+ return StaticCredentialsProvider.create(
+ AwsSessionCredentials.create(
+ asMap.get(ACCESS_KEY_ID),
+ asMap.get(SECRET_ACCESS_KEY),
+ asMap.get(SESSION_TOKEN)));
+ } else {
+ return StaticCredentialsProvider.create(
+ AwsBasicCredentials.create(asMap.get(ACCESS_KEY_ID), asMap.get(SECRET_ACCESS_KEY)));
+ }
} else if (typeName.equals(DefaultCredentialsProvider.class.getSimpleName())) {
return DefaultCredentialsProvider.create();
} else if (typeName.equals(EnvironmentVariableCredentialsProvider.class.getSimpleName())) {
@@ -158,10 +169,16 @@
// BEAM-11958 Use deprecated Jackson APIs to be compatible with older versions of jackson
typeSerializer.writeTypePrefixForObject(credentialsProvider, jsonGenerator);
if (credentialsProvider.getClass().equals(StaticCredentialsProvider.class)) {
- jsonGenerator.writeStringField(
- ACCESS_KEY_ID, credentialsProvider.resolveCredentials().accessKeyId());
- jsonGenerator.writeStringField(
- SECRET_ACCESS_KEY, credentialsProvider.resolveCredentials().secretAccessKey());
+ AwsCredentials credentials = credentialsProvider.resolveCredentials();
+ if (credentials.getClass().equals(AwsSessionCredentials.class)) {
+ AwsSessionCredentials sessionCredentials = (AwsSessionCredentials) credentials;
+ jsonGenerator.writeStringField(ACCESS_KEY_ID, sessionCredentials.accessKeyId());
+ jsonGenerator.writeStringField(SECRET_ACCESS_KEY, sessionCredentials.secretAccessKey());
+ jsonGenerator.writeStringField(SESSION_TOKEN, sessionCredentials.sessionToken());
+ } else {
+ jsonGenerator.writeStringField(ACCESS_KEY_ID, credentials.accessKeyId());
+ jsonGenerator.writeStringField(SECRET_ACCESS_KEY, credentials.secretAccessKey());
+ }
} else if (!SINGLETON_CREDENTIAL_PROVIDERS.contains(credentialsProvider.getClass())) {
throw new IllegalArgumentException(
"Unsupported AWS credentials provider type " + credentialsProvider.getClass());
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
index 2f0e038..9414e24 100644
--- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
@@ -33,6 +33,7 @@
import org.junit.runners.JUnit4;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
@@ -68,6 +69,20 @@
assertEquals(
credentialsProvider.resolveCredentials().secretAccessKey(),
deserializedCredentialsProvider.resolveCredentials().secretAccessKey());
+
+ AwsSessionCredentials sessionCredentials =
+ AwsSessionCredentials.create("key-id", "secret-key", "session-token");
+ credentialsProvider = StaticCredentialsProvider.create(sessionCredentials);
+ serializedCredentialsProvider = objectMapper.writeValueAsString(credentialsProvider);
+ deserializedCredentialsProvider =
+ objectMapper.readValue(serializedCredentialsProvider, AwsCredentialsProvider.class);
+
+ assertEquals(credentialsProvider.getClass(), deserializedCredentialsProvider.getClass());
+ AwsSessionCredentials deserializedCredentials =
+ (AwsSessionCredentials) deserializedCredentialsProvider.resolveCredentials();
+ assertEquals(sessionCredentials.accessKeyId(), deserializedCredentials.accessKeyId());
+ assertEquals(sessionCredentials.secretAccessKey(), deserializedCredentials.secretAccessKey());
+ assertEquals(sessionCredentials.sessionToken(), deserializedCredentials.sessionToken());
}
@Test
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index bbaf52c..29a6016 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -508,6 +508,13 @@
# in the proto representation of the graph.
pipeline.replace_all(DataflowRunner._NON_PORTABLE_PTRANSFORM_OVERRIDES)
+ # Always upload graph out-of-band when explicitly using runner v2 with
+ # use_portable_job_submission to avoid irrelevant large graph limits.
+ if (apiclient._use_unified_worker(debug_options) and
+ debug_options.lookup_experiment('use_portable_job_submission') and
+ not debug_options.lookup_experiment('upload_graph')):
+ debug_options.add_experiment("upload_graph")
+
# Add setup_options for all the BeamPlugin imports
setup_options = options.view_as(SetupOptions)
plugins = BeamPlugin.get_all_plugin_paths()
diff --git a/website/www/site/content/en/documentation/io/built-in/hcatalog.md b/website/www/site/content/en/documentation/io/built-in/hcatalog.md
index 9cf5d9a..22d4ffa 100644
--- a/website/www/site/content/en/documentation/io/built-in/hcatalog.md
+++ b/website/www/site/content/en/documentation/io/built-in/hcatalog.md
@@ -47,6 +47,7 @@
The destination table should exist beforehand as the transform will not create a new table if missing.
For example:
+
{{< highlight java >}}
Map<String, String> configProperties = new HashMap<String, String>();
configProperties.put("hive.metastore.uris","thrift://metastore-host:port");
@@ -60,7 +61,6 @@
.withPartition(partitionValues) //optional, may be specified if the table is partitioned
.withBatchSize(1024L)) //optional, assumes a default batch size of 1024 if none specified
{{< /highlight >}}
-
{{< highlight py >}}
# The Beam SDK for Python does not support HCatalogIO.
{{< /highlight >}}
