| { |
| "__inputs": [ |
| { |
| "name": "DS_BEAMPSQL", |
| "label": "BeamPSQL", |
| "description": "", |
| "type": "datasource", |
| "pluginId": "postgres", |
| "pluginName": "PostgreSQL" |
| } |
| ], |
| "__requires": [ |
| { |
| "type": "grafana", |
| "id": "grafana", |
| "name": "Grafana", |
| "version": "5.3.1" |
| }, |
| { |
| "type": "panel", |
| "id": "graph", |
| "name": "Graph", |
| "version": "5.0.0" |
| }, |
| { |
| "type": "datasource", |
| "id": "postgres", |
| "name": "PostgreSQL", |
| "version": "5.0.0" |
| }, |
| { |
| "type": "panel", |
| "id": "table", |
| "name": "Table", |
| "version": "5.0.0" |
| }, |
| { |
| "type": "panel", |
| "id": "text", |
| "name": "Text", |
| "version": "5.0.0" |
| } |
| ], |
| "annotations": { |
| "list": [ |
| { |
| "builtIn": 1, |
| "datasource": "-- Grafana --", |
| "enable": true, |
| "hide": true, |
| "iconColor": "rgba(0, 211, 255, 1)", |
| "limit": 100, |
| "name": "Annotations & Alerts", |
| "showIn": 0, |
| "type": "dashboard" |
| } |
| ] |
| }, |
| "editable": true, |
| "gnetId": null, |
| "graphTooltip": 0, |
| "id": null, |
| "links": [], |
| "panels": [ |
| { |
| "content": "This dashboard tracks Post-commit test reliability over-time.\n\n* [Post-commit test policies](https://beam.apache.org/contribute/postcommits-policies/)\n* [Existing test failure issues](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC)\n* [File a new test failure issue](https://s.apache.org/beam-test-failure)", |
| "gridPos": { |
| "h": 4, |
| "w": 24, |
| "x": 0, |
| "y": 0 |
| }, |
| "id": 11, |
| "links": [], |
| "mode": "markdown", |
| "title": "Dashboard guidelines", |
| "transparent": false, |
| "type": "text" |
| }, |
| { |
| "alert": { |
| "conditions": [ |
| { |
| "evaluator": { |
| "params": [ |
| 0.7 |
| ], |
| "type": "lt" |
| }, |
| "operator": { |
| "type": "and" |
| }, |
| "query": { |
| "params": [ |
| "A", |
| "5m", |
| "now" |
| ] |
| }, |
| "reducer": { |
| "params": [], |
| "type": "min" |
| }, |
| "type": "query" |
| } |
| ], |
| "executionErrorState": "alerting", |
| "frequency": "30m", |
| "handler": 1, |
| "name": "Post-commit reliability per week alert", |
| "noDataState": "no_data", |
| "notifications": [] |
| }, |
| "aliasColors": {}, |
| "bars": false, |
| "dashLength": 10, |
| "dashes": false, |
| "datasource": "${DS_BEAMPSQL}", |
| "decimals": 0, |
| "description": "Percent reliability of all post-commit job runs for a given week.\n\nUnreliability of a test suite impact developer productivity by forcing contributors to re-run tests. When tests are consistently unreliable, developers will simply ignore them.\n\nWe aim for >= 70% reliability per test suite.", |
| "fill": 0, |
| "gridPos": { |
| "h": 7, |
| "w": 24, |
| "x": 0, |
| "y": 4 |
| }, |
| "id": 6, |
| "legend": { |
| "alignAsTable": true, |
| "avg": false, |
| "current": true, |
| "hideEmpty": false, |
| "hideZero": true, |
| "max": false, |
| "min": false, |
| "rightSide": true, |
| "show": true, |
| "total": false, |
| "values": true |
| }, |
| "lines": true, |
| "linewidth": 1, |
| "links": [], |
| "nullPointMode": "null", |
| "percentage": false, |
| "pointradius": 2, |
| "points": false, |
| "renderer": "flot", |
| "seriesOverrides": [], |
| "spaceLength": 10, |
| "stack": false, |
| "steppedLine": false, |
| "targets": [ |
| { |
| "alias": "", |
| "format": "time_series", |
| "group": [], |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "SELECT\n DATE_TRUNC('week', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n job_name, time\n", |
| "refId": "A", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| } |
| ], |
| "thresholds": [ |
| { |
| "colorMode": "critical", |
| "fill": true, |
| "line": true, |
| "op": "lt", |
| "value": 0.7 |
| } |
| ], |
| "timeFrom": null, |
| "timeShift": null, |
| "title": "Post-commit reliability per week", |
| "tooltip": { |
| "shared": true, |
| "sort": 1, |
| "value_type": "individual" |
| }, |
| "type": "graph", |
| "xaxis": { |
| "buckets": null, |
| "mode": "time", |
| "name": null, |
| "show": true, |
| "values": [] |
| }, |
| "yaxes": [ |
| { |
| "decimals": 1, |
| "format": "percentunit", |
| "label": "% successful runs", |
| "logBase": 1, |
| "max": "1", |
| "min": "0", |
| "show": true |
| }, |
| { |
| "format": "short", |
| "label": null, |
| "logBase": 1, |
| "max": null, |
| "min": null, |
| "show": false |
| } |
| ], |
| "yaxis": { |
| "align": false, |
| "alignLevel": null |
| } |
| }, |
| { |
| "alert": { |
| "conditions": [ |
| { |
| "evaluator": { |
| "params": [ |
| 0.7 |
| ], |
| "type": "lt" |
| }, |
| "operator": { |
| "type": "and" |
| }, |
| "query": { |
| "params": [ |
| "A", |
| "5m", |
| "now" |
| ] |
| }, |
| "reducer": { |
| "params": [], |
| "type": "min" |
| }, |
| "type": "query" |
| } |
| ], |
| "executionErrorState": "alerting", |
| "frequency": "30m", |
| "handler": 1, |
| "name": "Post-commit reliability per day alert", |
| "noDataState": "no_data", |
| "notifications": [] |
| }, |
| "aliasColors": {}, |
| "bars": false, |
| "dashLength": 10, |
| "dashes": false, |
| "datasource": "${DS_BEAMPSQL}", |
| "decimals": 0, |
| "description": "Percent reliability of all post-commit job runs per-day.\n\nUnreliability of a test suite impact developer productivity by forcing contributors to re-run tests. When tests are consistently unreliable, developers will simply ignore them.\n\nWe aim for >= 70% reliability per test suite.", |
| "fill": 0, |
| "gridPos": { |
| "h": 12, |
| "w": 15, |
| "x": 0, |
| "y": 11 |
| }, |
| "id": 9, |
| "legend": { |
| "alignAsTable": true, |
| "avg": false, |
| "current": true, |
| "hideZero": true, |
| "max": false, |
| "min": false, |
| "rightSide": true, |
| "show": false, |
| "sideWidth": null, |
| "total": false, |
| "values": true |
| }, |
| "lines": true, |
| "linewidth": 1, |
| "links": [], |
| "nullPointMode": "null", |
| "percentage": false, |
| "pointradius": 2, |
| "points": false, |
| "renderer": "flot", |
| "seriesOverrides": [], |
| "spaceLength": 10, |
| "stack": false, |
| "steppedLine": false, |
| "targets": [ |
| { |
| "alias": "", |
| "format": "time_series", |
| "group": [], |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "SELECT\n DATE_TRUNC('day', build_timestamp) as time,\n avg(\n case \n when build_result = 'SUCCESS' then 1\n else 0\n end) as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as job_name\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name like '%_PR')\nGROUP BY\n time, job_name\norder BY\n job_name, time\n", |
| "refId": "A", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| } |
| ], |
| "thresholds": [ |
| { |
| "colorMode": "critical", |
| "fill": true, |
| "line": true, |
| "op": "lt", |
| "value": 0.7 |
| } |
| ], |
| "timeFrom": null, |
| "timeShift": null, |
| "title": "Post-commit reliability per day", |
| "tooltip": { |
| "shared": true, |
| "sort": 1, |
| "value_type": "individual" |
| }, |
| "type": "graph", |
| "xaxis": { |
| "buckets": null, |
| "mode": "time", |
| "name": null, |
| "show": true, |
| "values": [] |
| }, |
| "yaxes": [ |
| { |
| "decimals": 1, |
| "format": "percentunit", |
| "label": "% successful runs", |
| "logBase": 1, |
| "max": "1", |
| "min": "0", |
| "show": true |
| }, |
| { |
| "format": "short", |
| "label": null, |
| "logBase": 1, |
| "max": null, |
| "min": null, |
| "show": false |
| } |
| ], |
| "yaxis": { |
| "align": false, |
| "alignLevel": null |
| } |
| }, |
| { |
| "columns": [], |
| "datasource": "${DS_BEAMPSQL}", |
| "description": "List of jobs which have failed. Click on the job to view it in Jenkins.", |
| "fontSize": "100%", |
| "gridPos": { |
| "h": 12, |
| "w": 9, |
| "x": 15, |
| "y": 11 |
| }, |
| "hideTimeOverride": false, |
| "id": 8, |
| "links": [ |
| { |
| "includeVars": false, |
| "targetBlank": true, |
| "title": "Beam Jenkins", |
| "type": "absolute", |
| "url": "https://builds.apache.org/view/A-D/view/Beam/" |
| } |
| ], |
| "pageSize": null, |
| "scroll": true, |
| "showHeader": true, |
| "sort": { |
| "col": 0, |
| "desc": true |
| }, |
| "styles": [ |
| { |
| "alias": "Time", |
| "dateFormat": "YYYY-MM-DD HH:mm:ss", |
| "link": false, |
| "pattern": "Time", |
| "type": "date" |
| }, |
| { |
| "alias": "Build Url", |
| "colorMode": null, |
| "colors": [ |
| "rgba(245, 54, 54, 0.9)", |
| "rgba(237, 129, 40, 0.89)", |
| "rgba(50, 172, 45, 0.97)" |
| ], |
| "dateFormat": "YYYY-MM-DD HH:mm:ss", |
| "decimals": 2, |
| "link": true, |
| "linkTargetBlank": true, |
| "linkTooltip": "Link to Jenkins job.", |
| "linkUrl": "${__cell:raw}", |
| "mappingType": 1, |
| "pattern": "build_url", |
| "thresholds": [], |
| "type": "hidden", |
| "unit": "short" |
| }, |
| { |
| "alias": "Job Name", |
| "colorMode": null, |
| "colors": [ |
| "rgba(245, 54, 54, 0.9)", |
| "rgba(237, 129, 40, 0.89)", |
| "rgba(50, 172, 45, 0.97)" |
| ], |
| "dateFormat": "YYYY-MM-DD HH:mm:ss", |
| "decimals": 2, |
| "link": true, |
| "linkTargetBlank": true, |
| "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}", |
| "linkUrl": "${__cell_0:raw}", |
| "mappingType": 1, |
| "pattern": "job_name", |
| "thresholds": [], |
| "type": "string", |
| "unit": "short" |
| }, |
| { |
| "alias": "Build ID", |
| "colorMode": null, |
| "colors": [ |
| "rgba(245, 54, 54, 0.9)", |
| "rgba(237, 129, 40, 0.89)", |
| "rgba(50, 172, 45, 0.97)" |
| ], |
| "dateFormat": "YYYY-MM-DD HH:mm:ss", |
| "decimals": 0, |
| "link": true, |
| "linkTargetBlank": true, |
| "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}", |
| "linkUrl": "${__cell_0:raw}", |
| "mappingType": 1, |
| "pattern": "build_id", |
| "thresholds": [], |
| "type": "number", |
| "unit": "short" |
| }, |
| { |
| "alias": "Start Time", |
| "colorMode": null, |
| "colors": [ |
| "rgba(245, 54, 54, 0.9)", |
| "rgba(237, 129, 40, 0.89)", |
| "rgba(50, 172, 45, 0.97)" |
| ], |
| "dateFormat": "MM/DD/YY h:mm:ss a", |
| "decimals": 2, |
| "link": true, |
| "linkTargetBlank": true, |
| "linkTooltip": "View Jenkins job: ${__cell_1}_${__cell_2}", |
| "linkUrl": "${__cell_0:raw}", |
| "mappingType": 1, |
| "pattern": "build_timestamp", |
| "thresholds": [], |
| "type": "date", |
| "unit": "short" |
| } |
| ], |
| "targets": [ |
| { |
| "alias": "", |
| "format": "table", |
| "group": [], |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "SELECT \n build_url,\n job_name,\n build_id,\n build_timestamp\nFROM jenkins_builds\nWHERE \n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name LIKE '%_PR')\n AND NOT (build_result = 'SUCCESS')\nORDER BY \n build_timestamp", |
| "refId": "A", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| } |
| ], |
| "timeShift": null, |
| "title": "Failed builds", |
| "transform": "table", |
| "type": "table" |
| }, |
| { |
| "aliasColors": {}, |
| "bars": false, |
| "dashLength": 10, |
| "dashes": false, |
| "datasource": "${DS_BEAMPSQL}", |
| "decimals": 1, |
| "description": "Execution time for each post-commit job", |
| "fill": 0, |
| "gridPos": { |
| "h": 8, |
| "w": 15, |
| "x": 0, |
| "y": 23 |
| }, |
| "id": 5, |
| "legend": { |
| "alignAsTable": true, |
| "avg": false, |
| "current": true, |
| "max": false, |
| "min": false, |
| "rightSide": true, |
| "show": false, |
| "total": false, |
| "values": true |
| }, |
| "lines": true, |
| "linewidth": 1, |
| "links": [], |
| "nullPointMode": "null", |
| "percentage": false, |
| "pointradius": 5, |
| "points": false, |
| "renderer": "flot", |
| "seriesOverrides": [], |
| "spaceLength": 10, |
| "stack": false, |
| "steppedLine": false, |
| "targets": [ |
| { |
| "alias": "", |
| "format": "time_series", |
| "group": [], |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "SELECT\n build_timestamp as time,\n build_duration as value,\n substring(job_name from 'beam_PostCommit_#\"%#\"' for '#') as metric\nFROM\n jenkins_builds\nWHERE\n (build_timestamp BETWEEN $__timeFrom() AND $__timeTo())\n AND (job_name LIKE 'beam_PostCommit_%')\n AND NOT (job_name LIKE '%_PR')\nORDER BY\n job_name, time", |
| "refId": "A", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| } |
| ], |
| "thresholds": [], |
| "timeFrom": null, |
| "timeShift": null, |
| "title": "Post-commit job duration", |
| "tooltip": { |
| "shared": true, |
| "sort": 2, |
| "value_type": "individual" |
| }, |
| "type": "graph", |
| "xaxis": { |
| "buckets": null, |
| "mode": "time", |
| "name": null, |
| "show": true, |
| "values": [] |
| }, |
| "yaxes": [ |
| { |
| "decimals": null, |
| "format": "ms", |
| "label": "Average job duration", |
| "logBase": 1, |
| "max": null, |
| "min": "0", |
| "show": true |
| }, |
| { |
| "format": "short", |
| "label": null, |
| "logBase": 1, |
| "max": null, |
| "min": null, |
| "show": false |
| } |
| ], |
| "yaxis": { |
| "align": false, |
| "alignLevel": null |
| } |
| }, |
| { |
| "aliasColors": {}, |
| "bars": true, |
| "dashLength": 10, |
| "dashes": false, |
| "datasource": "${DS_BEAMPSQL}", |
| "decimals": 0, |
| "description": "Tracks the count of test failure JIRA issues currently open.", |
| "fill": 3, |
| "gridPos": { |
| "h": 8, |
| "w": 9, |
| "x": 15, |
| "y": 23 |
| }, |
| "id": 14, |
| "legend": { |
| "alignAsTable": false, |
| "avg": false, |
| "current": false, |
| "max": false, |
| "min": false, |
| "rightSide": false, |
| "show": true, |
| "total": false, |
| "values": false |
| }, |
| "lines": false, |
| "linewidth": 1, |
| "links": [ |
| { |
| "targetBlank": true, |
| "title": "Jira tickets", |
| "type": "absolute", |
| "url": "https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20%3D%20test-failures%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC" |
| } |
| ], |
| "nullPointMode": "null", |
| "percentage": false, |
| "pointradius": 5, |
| "points": false, |
| "renderer": "flot", |
| "seriesOverrides": [ |
| { |
| "alias": "total_open", |
| "color": "#eab839" |
| }, |
| { |
| "alias": "currently_failing", |
| "color": "#bf1b00" |
| } |
| ], |
| "spaceLength": 10, |
| "stack": false, |
| "steppedLine": false, |
| "targets": [ |
| { |
| "format": "time_series", |
| "group": [], |
| "hide": false, |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "with days as (select date_trunc('day', dd) as day from generate_series( $__timeFrom()::timestamp, $__timeTo()::timestamp, '1 day'::interval) as dd),\n knowndays as (SELECT days.day, count(*) as total_open\n FROM jira_issues, days\n WHERE jira_issues.created < days.day AND (jira_issues.resolutiondate > days.day OR jira_issues.resolutiondate is null)\n GROUP BY days.day\n ORDER BY days.day)\nselect days.day as time, greatest(knowndays.total_open, 0) as total_open\nfrom days left outer join knowndays\non days.day = knowndays.day", |
| "refId": "A", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| }, |
| { |
| "format": "time_series", |
| "group": [], |
| "hide": false, |
| "metricColumn": "none", |
| "rawQuery": true, |
| "rawSql": "with days as (select date_trunc('day', dd) as day from generate_series( $__timeFrom()::timestamp, $__timeTo()::timestamp, '1 day'::interval) as dd),\n knowndays as (SELECT days.day, count(*) as currently_failing\n FROM jira_issues, days\n WHERE jira_issues.created < days.day AND (jira_issues.resolutiondate > days.day OR jira_issues.resolutiondate is null) AND (jira_issues.labels LIKE '%currently-failing%')\n GROUP BY days.day\n ORDER BY days.day)\nselect days.day as time, greatest(knowndays.currently_failing, 0) as currently_failing\nfrom days left outer join knowndays\non days.day = knowndays.day", |
| "refId": "D", |
| "select": [ |
| [ |
| { |
| "params": [ |
| "value" |
| ], |
| "type": "column" |
| } |
| ] |
| ], |
| "timeColumn": "time", |
| "where": [ |
| { |
| "name": "$__timeFilter", |
| "params": [], |
| "type": "macro" |
| } |
| ] |
| } |
| ], |
| "thresholds": [], |
| "timeFrom": null, |
| "timeShift": null, |
| "title": "Test Failure JIRA issues", |
| "tooltip": { |
| "shared": true, |
| "sort": 0, |
| "value_type": "individual" |
| }, |
| "type": "graph", |
| "xaxis": { |
| "buckets": null, |
| "mode": "time", |
| "name": null, |
| "show": true, |
| "values": [] |
| }, |
| "yaxes": [ |
| { |
| "decimals": 0, |
| "format": "short", |
| "label": "# of JIRA issues", |
| "logBase": 1, |
| "max": null, |
| "min": "0", |
| "show": true |
| }, |
| { |
| "format": "short", |
| "label": null, |
| "logBase": 1, |
| "max": null, |
| "min": null, |
| "show": false |
| } |
| ], |
| "yaxis": { |
| "align": false, |
| "alignLevel": null |
| } |
| } |
| ], |
| "refresh": false, |
| "schemaVersion": 16, |
| "style": "dark", |
| "tags": [], |
| "templating": { |
| "list": [] |
| }, |
| "time": { |
| "from": "now-30d", |
| "to": "now" |
| }, |
| "timepicker": { |
| "hidden": false, |
| "refresh_intervals": [ |
| "1m", |
| "5m", |
| "15m", |
| "30m", |
| "1h", |
| "2h", |
| "1d" |
| ], |
| "time_options": [ |
| "5m", |
| "15m", |
| "1h", |
| "6h", |
| "12h", |
| "24h", |
| "2d", |
| "7d", |
| "30d" |
| ] |
| }, |
| "timezone": "", |
| "title": "Post-commit Test Reliability", |
| "uid": "D81lW0pmk", |
| "version": 45 |
| } |
