Merge pull request #380 from Startrekzky/go_dashboard
feat(grafana dashboard): add PR review rounds
diff --git a/.env.example b/.env.example
index 77ba225..1709d32 100644
--- a/.env.example
+++ b/.env.example
@@ -1,13 +1,14 @@
-###################################
-# Lake Database Connection String #
-###################################
+#############
+# Lake core #
+#############
-DB_URL=merico:merico@tcp(localhost:3306)/lake?charset=utf8mb4&loc=Asia%2fShanghai&parseTime=True
+# Lake plugin dir, absolute path or relative path
+PLUGIN_DIR=bin/plugins
-#################
-# Lake REST API #
-#################
+# Lake Database Connection String
+DB_URL=merico:merico@tcp(mysql:3306)/lake?charset=utf8mb4&parseTime=True
+# Lake REST API
PORT=:8080
MODE=debug
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index a95b61c..79b5306 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -5,7 +5,7 @@
e2e-mysql:
strategy:
matrix:
- go-version: [1.15.x]
+ go-version: [1.16.x]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
env:
@@ -31,5 +31,5 @@
redis-version: '6.x'
- name: Test
run: |
- sh ./scripts/compile-plugins.sh
- sh ./scripts/e2e-test.sh
+ cp .env.example .env
+ make e2e-test
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a29d68a..b134216 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,7 +4,7 @@
test:
strategy:
matrix:
- go-version: [1.15.x]
+ go-version: [1.16.x]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
@@ -16,5 +16,4 @@
uses: actions/checkout@v2
- name: Test
run: |
- sh ./scripts/compile-plugins.sh
- sh ./scripts/unit-test.sh
+ make unit-test
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
new file mode 100644
index 0000000..b9e8c43
--- /dev/null
+++ b/ARCHITECTURE.md
@@ -0,0 +1,16 @@
+# Architecture Layers
+
+## Stack (from low to high)
+
+1. config
+2. logger
+3. models
+4. plugins
+5. services
+6. api / cli
+
+## Rules
+
+1. Higher layer calls lower layer, not the other way around
+2. Whenever lower layer neeeds something from higher layer, a interface should be introduced for decoupling
+3. Components should be initialized in a low to high order during bootstraping
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 1f43fa3..6a517af 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,21 +1,25 @@
-FROM mericodev/lake-builder:0.0.1 as builder
+FROM mericodev/lake-builder:0.0.2 as builder
# docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake .
ARG GOPROXY=
WORKDIR /app
COPY . /app
-RUN CGO_ENABLE=1 GOOS=linux go build -o lake && sh scripts/compile-plugins.sh
+RUN rm -rf /app/bin
+
+ENV GOBIN=/app/bin
+
+RUN CGO_ENABLE=1 GOOS=linux go build -o bin/lake && sh scripts/compile-plugins.sh
RUN go install ./cmd/lake-cli/
FROM alpine:edge
-EXPOSE 8080
-COPY --from=builder /app/lake /bin/app/lake
-COPY --from=builder /app/plugins/ /bin/app/plugins
-# copy zoneinfo.zip into container
-# zoneinfo.zip is from $GOROOT/lib/time/zoneinfo.zip
-COPY --from=builder /app/zoneinfo.zip /usr/local/go/lib/time/zoneinfo.zip
-COPY --from=builder /go/bin/lake-cli /bin/lake-cli
-WORKDIR /bin/app
-CMD ["/bin/sh", "-c", "./lake"]
+EXPOSE 8080
+WORKDIR /app
+
+COPY --from=builder /app/bin /app/bin
+COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
+
+ENV PATH="/app/bin:${PATH}"
+
+CMD ["lake"]
diff --git a/Makefile b/Makefile
index dbf2cc3..6901769 100644
--- a/Makefile
+++ b/Makefile
@@ -5,15 +5,21 @@
hello:
echo "Hello"
-build:
- go build
+build-plugin:
+ @sh scripts/compile-plugins.sh
-dev:
- @sh ./scripts/dev.sh
+build: build-plugin
+ go build -o bin/lake
+
+dev: build
+ bin/lake
run:
go run main.go
+configure:
+ cd config-ui; yarn; npm run dev;
+
compose:
docker-compose up grafana
@@ -29,12 +35,14 @@
test: unit-test e2e-test
-unit-test:
- @sh ./scripts/unit-test.sh
+unit-test: build
+ ENV_FILE=`pwd`/.env go test -v $$(go list ./... | grep -v /test/)
-e2e-test:
- @sh ./scripts/e2e-test.sh
+e2e-test: build
+ ENV_FILE=`pwd`/.env go test -v ./test/...
lint:
golangci-lint run
+clean:
+ @rm -rf bin
diff --git a/README.md b/README.md
index a4f8793..456e993 100644
--- a/README.md
+++ b/README.md
@@ -26,18 +26,16 @@
Section | Description | Documentation Link
:------------ | :------------- | :-------------
Data Sources | Links to specific plugin usage & details | [View Section](#data-source-plugins)
-Grafana | How to visualize the data | [View Section](#grafana)
-Requirements | Underlying software used | [View Section](#requirements)
-Setup | Steps to setup the project | [View Section](#user-setup)
-Migrations | Commands for running migrations | [View Section](#migrations)
+User Setup | Steps to run the project as a user | [View Section](#user-setup)
+Developer Setup | How to setup dev environment | [View Section](#dev-setup)
Tests | Commands for running tests | [View Section](#tests)
-⚠️ (WIP) Build a Plugin | Details on how to make your own | [Link](plugins/README.md)
-⚠️ (WIP) Add Plugin Metrics | Guide to adding plugin metrics | [Link](plugins/HOW-TO-ADD-METRICS.md)
+Grafana | How to visualize the data | [View Section](#grafana)
+Build a Plugin | Details on how to make your own | [Link](plugins/README.md)
+Add Plugin Metrics | Guide to adding plugin metrics | [Link](plugins/HOW-TO-ADD-METRICS.md)
Contributing | How to contribute to this repo | [Link](CONTRIBUTING.md)
-
-## ⚠️ (WIP) Data Sources We Currently Support<a id="data-source-plugins"></a>
+## Data Sources We Currently Support<a id="data-source-plugins"></a>
Below is a list of _data source plugins_ used to collect & enrich data from specific sources. Each have a `README.md` file with basic setup, troubleshooting and metrics info.
@@ -47,24 +45,8 @@
------------ | ------------- | -------------
Jira | Metrics, Generating API Token, Find Project/Board ID | [Link](plugins/jira/README.md)
Gitlab | Metrics, Generating API Token | [Link](plugins/gitlab/README.md)
-Jenkins | Metrics | [Link](plugins/jenkins/README.md)
+Jenkins | Metrics, Generating API Token | [Link](plugins/jenkins/README.md)
-## Grafana<a id="grafana"></a>
-
-We use <a href="https://grafana.com/" target="_blank">Grafana</a> as a visualization tool to build charts for the data stored in our database. Using SQL queries we can add panels to build, save, and edit customized dashboards.
-
-All the details on provisioning, and customizing a dashboard can be found in the [Grafana Doc](docs/GRAFANA.md)
-
----
-
-## Development Requirements<a id="requirements"></a>
-
-- <a href="https://docs.docker.com/get-docker" target="_blank">Docker</a>
-- <a href="https://golang.org/doc/install" target="_blank">Golang</a>
-- Make
- - Mac (Already installed)
- - Windows: [Download](http://gnuwin32.sourceforge.net/packages/make.htm)
- - Ubuntu: `sudo apt-get install build-essential`
## User setup<a id="user-setup"></a>
@@ -96,7 +78,7 @@
3. Launch `docker-compose`
```shell
- docker-compose up
+ make compose
```
4. Create a http request to trigger data collect tasks, please replace your [gitlab projectId](plugins/gitlab/README.md#finding-project-id) and [jira boardId](plugins/jira/README.md#find-board-id) in the request body. This can take up to 20 minutes for large projects. (gitlab 10k+ commits or jira 5k+ issues)
@@ -131,8 +113,19 @@
Otherwise, if you just want to use the cron job, please check `docker-compose` version at [here](./devops/sync/README.md)
-## Development Setup<a id="development-setup"></a>
+## Developer Setup<a id="dev-setup"></a>
+
+### Requirements
+
+- <a href="https://docs.docker.com/get-docker" target="_blank">Docker</a>
+- <a href="https://golang.org/doc/install" target="_blank">Golang</a>
+- Make
+ - Mac (Already installed)
+ - Windows: [Download](http://gnuwin32.sourceforge.net/packages/make.htm)
+ - Ubuntu: `sudo apt-get install build-essential`
+
+### How to setup dev environment
1. Navigate to where you would like to install this project and clone the repository
```sh
@@ -179,23 +172,7 @@
}]'
```
-7. Collect & enrich data from selected sources and plugins
-
- _These plugins can be selected from the above list ([View List](#data-source-plugins)), and options for them will be outlined in their specific document._
-
- **Example:**
-
- ```sh
- curl -XPOST 'localhost:8080/source' \
- -H 'Content-Type: application/json' \
- -d '[{
- "plugin": "Jira",
- "options": {}
-
- }]'
- ```
-
-8. Visualize the data in the Grafana Dashboard
+7. Visualize the data in the Grafana Dashboard
_From here you can see existing data visualized from collected & enriched data_
@@ -204,18 +181,16 @@
- For more info on working with Grafana in Dev Lake see [Grafana Doc](docs/GRAFANA.md)
-## ⚠️ (WIP) Migrations<a id="migrations"></a>
-
-- Make a migration: `<>`
-- Migrate your DB: `<>`
-- Undo a migration: `<>`
-
## Tests<a id="tests"></a>
-Sample tests can be found in `/test/example`
-
To run the tests: `make test`
+## Grafana<a id="grafana"></a>
+
+We use <a href="https://grafana.com/" target="_blank">Grafana</a> as a visualization tool to build charts for the data stored in our database. Using SQL queries we can add panels to build, save, and edit customized dashboards.
+
+All the details on provisioning, and customizing a dashboard can be found in the [Grafana Doc](docs/GRAFANA.md)
+
## Contributing
[CONTRIBUTING.md](CONTRIBUTING.md)
@@ -224,19 +199,6 @@
Message us on <a href="https://discord.com/invite/83rDG6ydVZ" target="_blank">Discord</a>
-# Architecture Layers
-## Stack (from low to high)
-1. config
-2. logger
-3. models
-4. plugins
-5. services
-6. api / cli
-## Rules
-
-1. Higher layer calls lower layer, not the other way around
-2. Whenever lower layer neeeds sth from higher layer, a interface should be introduced for decoupling
-3. Components should be initialized in a low to high order during bootstraping
diff --git a/config-ui/.dockerignore b/config-ui/.dockerignore
deleted file mode 100644
index 3c3629e..0000000
--- a/config-ui/.dockerignore
+++ /dev/null
@@ -1 +0,0 @@
-node_modules
diff --git a/config-ui/Dockerfile b/config-ui/Dockerfile
new file mode 100644
index 0000000..f9c5970
--- /dev/null
+++ b/config-ui/Dockerfile
@@ -0,0 +1,11 @@
+FROM node:14-alpine
+
+COPY package.json /src/package.json
+WORKDIR /src
+RUN npm i
+
+COPY . /src
+
+CMD ["npm", "run", "dev"]
+
+EXPOSE 4000
diff --git a/config-ui/components/Content.js b/config-ui/components/Content.js
new file mode 100644
index 0000000..95b91aa
--- /dev/null
+++ b/config-ui/components/Content.js
@@ -0,0 +1,8 @@
+import { Button, Card, Elevation, PopoverPosition } from "@blueprintjs/core"
+import styles from '../styles/Content.module.css'
+
+const Content = ({children}) => {
+ return <>{children}</>
+}
+
+export default Content
diff --git a/config-ui/components/Nav.js b/config-ui/components/Nav.js
new file mode 100644
index 0000000..c5a3bb4
--- /dev/null
+++ b/config-ui/components/Nav.js
@@ -0,0 +1,24 @@
+import {
+ Alignment,
+ Button,
+ Navbar,
+ Icon,
+} from '@blueprintjs/core'
+import styles from '../styles/Nav.module.css'
+
+const Nav = () => {
+
+ return <Navbar className={styles.navbar}>
+ <Navbar.Group align={Alignment.RIGHT}>
+ <a href="https://github.com/merico-dev/lake" target="_blank" className={styles.navIconLink}>
+ <Icon icon="git-branch" size={16} />
+ </a>
+ <Navbar.Divider />
+ <a href="mailto:hello@merico.dev" target="_blank" className={styles.navIconLink}>
+ <Icon icon="envelope" size={16} />
+ </a>
+ </Navbar.Group>
+ </Navbar>
+}
+
+export default Nav
diff --git a/config-ui/components/Sidebar.js b/config-ui/components/Sidebar.js
new file mode 100644
index 0000000..e66ca20
--- /dev/null
+++ b/config-ui/components/Sidebar.js
@@ -0,0 +1,24 @@
+import { Button, Card, Elevation, Icon } from "@blueprintjs/core"
+import styles from '../styles/Sidebar.module.css'
+
+const Sidebar = () => {
+ return <Card interactive={false} elevation={Elevation.ZERO} className={styles.card}>
+
+ <img src="/logo.svg" className={styles.logo} />
+ <a href="http://localhost:3002" target="_blank" className={styles.dashboardBtnLink}>
+ <Button icon="grouped-bar-chart" outlined={true} large={true} className={styles.dashboardBtn}>View Dashboards</Button>
+ </a>
+
+ <ul className={styles.sidebarMenu}>
+ <a href="/">
+ <li>
+ <Icon icon="layout-grid" size={16} className={styles.sidebarMenuListIcon} />
+ Configuration
+ </li>
+ <div className={styles.sidebarMenuDash}></div>
+ </a>
+ </ul>
+ </Card>
+}
+
+export default Sidebar
diff --git a/config-ui/package.json b/config-ui/package.json
index e542fee..08ca60a 100644
--- a/config-ui/package.json
+++ b/config-ui/package.json
@@ -9,6 +9,7 @@
"lint": "next lint"
},
"dependencies": {
+ "@blueprintjs/core": "^3.49.1",
"dotenv": "^10.0.0",
"next": "11.1.2",
"react": "17.0.2",
diff --git a/config-ui/pages/_app.js b/config-ui/pages/_app.js
index 1e1cec9..79ee025 100644
--- a/config-ui/pages/_app.js
+++ b/config-ui/pages/_app.js
@@ -1,5 +1,8 @@
+import 'normalize.css'
+import '@blueprintjs/core/lib/css/blueprint.css'
import '../styles/globals.css'
+
function MyApp({ Component, pageProps }) {
return <Component {...pageProps} />
}
diff --git a/config-ui/pages/index.js b/config-ui/pages/index.js
index 5a5745c..59be636 100644
--- a/config-ui/pages/index.js
+++ b/config-ui/pages/index.js
@@ -2,108 +2,259 @@
import { useState } from 'react'
import dotenv from 'dotenv'
import path from 'path'
+import * as fs from 'fs/promises'
+import { existsSync } from 'fs';
import styles from '../styles/Home.module.css'
+import { FormGroup, InputGroup, Button } from "@blueprintjs/core"
+import Nav from '../components/Nav'
+import Sidebar from '../components/Sidebar'
+import Content from '../components/Content'
export default function Home(props) {
const { env } = props
- const [dbUrl, setDbUrl] = useState('')
- const [port, setPort] = useState('')
- const [mode, setMode] = useState('')
+ const [dbUrl, setDbUrl] = useState(env.DB_URL)
+ const [port, setPort] = useState(env.PORT)
+ const [mode, setMode] = useState(env.MODE)
+ const [jiraEndpoint, setJiraEndpoint] = useState(env.JIRA_ENDPOINT)
+ const [jiraBasicAuthEncoded, setJiraBasicAuthEncoded] = useState(env.JIRA_BASIC_AUTH_ENCODED)
+ const [jiraIssueEpicKeyField, setJiraIssueEpicKeyField] = useState(env.JIRA_ISSUE_EPIC_KEY_FIELD)
+ const [gitlabEndpoint, setGitlabEndpoint] = useState(env.GITLAB_ENDPOINT)
+ const [gitlabAuth, setGitlabAuth] = useState(env.GITLAB_AUTH)
function updateEnv(key, value) {
- fetch(`http://localhost:4000/api/setenv/${key}/${value}`)
- alert('updated')
+ fetch(`http://localhost:4000/api/setenv/${key}/${encodeURIComponent(value)}`)
+ }
+
+ function saveAll(e) {
+ e.preventDefault()
+ updateEnv('DB_URL', dbUrl)
+ updateEnv('PORT', port)
+ updateEnv('MODE', mode)
+ updateEnv('JIRA_ENDPOINT', jiraEndpoint)
+ updateEnv('JIRA_BASIC_AUTH_ENCODED', jiraBasicAuthEncoded)
+ updateEnv('JIRA_ISSUE_EPIC_KEY_FIELD', jiraIssueEpicKeyField)
+ updateEnv('GITLAB_ENDPOINT', gitlabEndpoint)
+ updateEnv('GITLAB_AUTH', gitlabAuth)
+ alert('Config file updated, please restart devlake to apply new configuration')
}
return (
<div className={styles.container}>
<Head>
- <title>Create Next App</title>
+ <title>Devlake Config-UI</title>
<meta name="description" content="Lake: Config" />
<link rel="icon" href="/favicon.ico" />
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin />
+ <link href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600&display=swap" rel="stylesheet" />
+ <link href="https://fonts.googleapis.com/css2?family=Rubik:wght@500;600&display=swap" rel="stylesheet" />
</Head>
- <main className={styles.main}>
+ <Nav />
+ <Sidebar />
+ <Content>
+ <main className={styles.main}>
- <img src="/logo.svg" className={styles.logo} />
+ <div className={styles.headlineContainer}>
+ <h1>Configuration</h1>
+ <p className={styles.description}>Configure your <code className={styles.code}>.env</code> file values</p>
+ </div>
- <p className={styles.description}>Configure your <code className={styles.code}>.env</code> file values</p>
+ <form className={styles.form}>
+ <div className={styles.headlineContainer}>
+ <h2 className={styles.headline}>Main Database Connection</h2>
+ <p className={styles.description}>Settings for the mySQL database</p>
+ </div>
- <div className={styles.formContainer}>
- <h3 className={styles.headline}>Main Database Connection</h3>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="DB_URL"
+ inline={true}
+ labelFor="db-url"
+ helperText="The URL Connection string to the database"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="db-url"
+ placeholder="Enter DB Connection String"
+ defaultValue={dbUrl}
+ onChange={(e) => setDbUrl(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>DB_URL</label>
- <input className={styles.input} type="text" onChange={(e) => setDbUrl(e.target.value)} defaultValue={env.DB_URL} />
- <button className={styles.button} onClick={() => updateEnv('DB_URL', dbUrl)}>save</button>
- </div>
+ <div className={styles.headlineContainer}>
+ <h2 className={styles.headline}>REST Configuration</h2>
+ <p className={styles.description}>Configure main REST Settings</p>
+ </div>
- <div className={styles.formContainer}>
- <h3 className={styles.headline}>REST Configuration</h3>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="PORT"
+ inline={true}
+ labelFor="port"
+ helperText="The main port for the REST server"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="port"
+ placeholder="Enter Port eg. :8080"
+ defaultValue={port}
+ onChange={(e) => setPort(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>PORT</label>
- <input className={styles.input} type="text" onChange={(e) => setPort(e.target.value)} defaultValue={env.PORT} />
- <button className={styles.button} onClick={() => updateEnv('PORT', port)}>save</button>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="MODE"
+ inline={true}
+ labelFor="mode"
+ helperText="The development mode for the server"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="mode"
+ placeholder="Enter Mode eg. debug"
+ defaultValue={mode}
+ onChange={(e) => setMode(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>MODE</label>
- <input className={styles.input} type="text" onChange={(e) => setMode(e.target.value)} defaultValue={env.MODE} />
- <button className={styles.button} onClick={() => updateEnv('MODE', mode)}>save</button>
- </div>
+ <div className={styles.headlineContainer}>
+ <h2 className={styles.headline}>Jira Configuration</h2>
+ <p className={styles.description}>Jira Account and config settings</p>
+ </div>
- <div className={styles.formContainer}>
- <h3 className={styles.headline}>Jira Configuration</h3>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="JIRA_ENDPOINT"
+ inline={true}
+ labelFor="jira-endpoint"
+ helperText="Your custom url endpoint for Jira"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="jira-endpoint"
+ placeholder="Enter Jira endpoint eg. https://merico.atlassian.net"
+ defaultValue={jiraEndpoint}
+ onChange={(e) => setJiraEndpoint(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>JIRA_ENDPOINT</label>
- <input className={styles.input} type="text" onChange={(e) => setJiraEndpoint(e.target.value)} defaultValue={env.JIRA_ENDPOINT} />
- <button className={styles.button} onClick={() => updateEnv('JIRA_ENDPOINT', jiraEndpoint)}>save</button>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="JIRA_BASIC_AUTH_ENCODED"
+ inline={true}
+ labelFor="jira-basic-auth"
+ helperText="Your encoded Jira auth token"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="jira-basic-auth"
+ placeholder="Enter Jira Auth eg. EJrLG8DNeXADQcGOaaaX4B47"
+ defaultValue={jiraBasicAuthEncoded}
+ onChange={(e) => setJiraBasicAuthEncoded(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>JIRA_BASIC_AUTH_ENCODED</label>
- <input className={styles.input} type="text" onChange={(e) => setJiraBasicAuthEncoded(e.target.value)} defaultValue={env.JIRA_BASIC_AUTH_ENCODED} />
- <button className={styles.button} onClick={() => updateEnv('JIRA_BASIC_AUTH_ENCODED', jiraBasicAuthEncoded)}>save</button>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="JIRA_ISSUE_EPIC_KEY_FIELD"
+ inline={true}
+ labelFor="jira-epic-key"
+ helperText="Your custom epic key field (optional)"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="jira-epic-key"
+ placeholder="Enter Jira epic key field"
+ defaultValue={jiraIssueEpicKeyField}
+ onChange={(e) => setJiraIssueEpicKeyField(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>JIRA_ISSUE_EPIC_KEY_FIELD</label>
- <input className={styles.input} type="text" onChange={(e) => setJiraIssueEpicKeyField(e.target.value)} defaultValue={env.JIRA_ISSUE_EPIC_KEY_FIELD} />
- <button className={styles.button} onClick={() => updateEnv('JIRA_ISSUE_EPIC_KEY_FIELD', jiraIssueEpicKeyField)}>save</button>
- </div>
+ <div className={styles.headlineContainer}>
+ <h2 className={styles.headline}>Gitlab Configuration</h2>
+ <p className={styles.description}>Gitlab account and config settings</p>
+ </div>
- <div className={styles.formContainer}>
- <h3 className={styles.headline}>Gitlab Configuration</h3>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="GITLAB_ENDPOINT"
+ inline={true}
+ labelFor="gitlab-endpoint"
+ helperText="Gitlab API Endpoint"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="gitlab-endpoint"
+ placeholder="Enter Gitlab API endpoint"
+ defaultValue={gitlabEndpoint}
+ onChange={(e) => setGitlabEndpoint(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>GITLAB_ENDPOINT</label>
- <input className={styles.input} type="text" onChange={(e) => setGitlabEndpoint(e.target.value)} defaultValue={env.GITLAB_ENDPOINT} />
- <button className={styles.button} onClick={() => updateEnv('GITLAB_ENDPOINT', gitlabEndpoint)}>save</button>
- </div>
+ <div className={styles.formContainer}>
+ <FormGroup
+ label="GITLAB_AUTH"
+ inline={true}
+ labelFor="gitlab-auth"
+ helperText="Gitlab Auth Token"
+ className={styles.formGroup}
+ contentClassName={styles.formGroup}
+ >
+ <InputGroup
+ id="gitlab-auth"
+ placeholder="Enter Gitlab Auth Token eg. uJVEDxabogHbfFyu2riz"
+ defaultValue={gitlabAuth}
+ onChange={(e) => setGitlabAuth(e.target.value)}
+ className={styles.input}
+ />
+ </FormGroup>
+ </div>
- <div className={styles.formContainer}>
- <label className={styles.label}>GITLAB_AUTH</label>
- <input className={styles.input} type="text" onChange={(e) => setGitlabAuth(e.target.value)} defaultValue={env.GITLAB_AUTH} />
- <button className={styles.button} onClick={() => updateEnv('GITLAB_AUTH', gitlabAuth)}>save</button>
- </div>
-
- </main>
+ <Button type="submit" outlined={true} large={true} className={styles.saveBtn} onClick={saveAll}>Save Config</Button>
+ </form>
+ </main>
+ </Content>
</div>
)
}
export async function getStaticProps() {
- const fs = require('fs').promises
+ // const fs = require('fs').promises
- const filePath = path.join(process.cwd(), 'data', '../../config-ui/.env')
+ const filePath = process.env.ENV_FILEPATH || path.join(process.cwd(), 'data', '../../.env')
+ const exist = existsSync(filePath);
+ if (!exist) {
+ return {
+ props: {
+ env: {},
+ }
+ }
+ }
const fileData = await fs.readFile(filePath)
const env = dotenv.parse(fileData)
diff --git a/config-ui/styles/Content.module.css b/config-ui/styles/Content.module.css
new file mode 100644
index 0000000..00bb437
--- /dev/null
+++ b/config-ui/styles/Content.module.css
@@ -0,0 +1,4 @@
+.card {
+ background: #fff;
+ width: 100%;
+}
diff --git a/config-ui/styles/Home.module.css b/config-ui/styles/Home.module.css
index d9466ab..b4945e4 100644
--- a/config-ui/styles/Home.module.css
+++ b/config-ui/styles/Home.module.css
@@ -1,13 +1,12 @@
.container {
- padding: 0 0.5rem;
display: flex;
- flex-direction: column;
- justify-content: center;
- align-items: center;
+ justify-content: space-evenly;
+ padding-top: 50px;
+ min-height: 100vh;
}
.main {
- padding: 5rem 0;
+ padding: 4rem 4rem 4rem calc(250px + 4rem);
flex: 1;
display: flex;
flex-direction: column;
@@ -16,7 +15,8 @@
}
.description {
- margin-bottom: 2rem;
+ display: block;
+ margin-bottom: 1.6rem;
}
.logo {
@@ -28,12 +28,21 @@
margin-right: 0.2rem;
}
-.label {
- margin-right: 1rem;
+.input, .form {
+ width: 100%;
}
-.input {
- flex: 1;
+.formGroup {
+ width: 100%;
+ max-width: 600px;
+}
+
+.formGroup div {
+ color: #717484 !important;
+}
+
+.formGroup label {
+ color: #4B4D58;
}
.button {
@@ -43,11 +52,30 @@
.formContainer {
display: flex;
width: 100%;
- min-width: 700px;
align-items: center;
margin-bottom: 0.5rem;
}
+.headlineContainer {
+ width: 100%;
+}
+
+.headlineContainer h1 {
+ margin: 0 0 0.5rem;
+}
+
.headline {
margin-top: 1.5rem;
+ margin-bottom: 0.25rem;
+}
+
+.saveBtn {
+ margin: 2rem auto 0 0;
+ display: block;
+ background-color: #717484 !important;
+ color: #fff !important;
+}
+
+.saveBtn:hover {
+ opacity: 0.9;
}
diff --git a/config-ui/styles/Nav.module.css b/config-ui/styles/Nav.module.css
new file mode 100644
index 0000000..895e537
--- /dev/null
+++ b/config-ui/styles/Nav.module.css
@@ -0,0 +1,12 @@
+.navbar {
+ position: fixed;
+ top: 0;
+ border-radius: 0;
+ background: #F4F4F6;
+ box-shadow: none;
+ z-index: 0;
+}
+
+.navIconLink:hover {
+ color: #E8471C;
+}
diff --git a/config-ui/styles/Sidebar.module.css b/config-ui/styles/Sidebar.module.css
new file mode 100644
index 0000000..aad5c28
--- /dev/null
+++ b/config-ui/styles/Sidebar.module.css
@@ -0,0 +1,71 @@
+.card {
+ width: 250px;
+ border-radius: 0;
+ background: #F4F4F6;
+ z-index: 1;
+ box-shadow: none;
+ position: fixed;
+ height: 100vh;
+ left: 0;
+}
+
+.logo {
+ max-width: 60px;
+ display: block;
+ margin: -2rem auto 2.5rem;
+}
+
+.dashboardBtn {
+ margin: 0 auto;
+ display: block;
+ background-color: #E8471C !important;
+ color: #fff !important;
+}
+
+.dashboardBtn:hover {
+ opacity: 0.9;
+}
+
+.dashboardBtnLink:hover {
+ text-decoration: none;
+}
+
+.dashboardBtn span svg {
+ fill: #ff7b56 !important;
+}
+
+.sidebarMenu {
+ margin-top: 2.5rem;
+ list-style-type: none;
+ padding-left: 0;
+ margin-left: -20px;
+ width: calc(100% + 40px);
+}
+
+.sidebarMenu a {
+ background: #FFDBC2;
+ display: block;
+ padding: 12px 20px;
+ color: #E8471C;
+ position: relative;
+ text-decoration: none;
+}
+
+.sidebarMenu a:hover {
+ background-color: #ffdac6;
+}
+
+.sidebarMenuDash {
+ height: 100%;
+ position: absolute;
+ background-color: #F8C3B5;
+ top: 0;
+ right: 0;
+ width: 3px;
+ border-top-left-radius: 2px;
+ border-bottom-left-radius: 2px;
+}
+
+.sidebarMenuListIcon {
+ margin-right: 0.75rem;
+}
diff --git a/config-ui/styles/globals.css b/config-ui/styles/globals.css
index e5e2dcc..d6ecbf6 100644
--- a/config-ui/styles/globals.css
+++ b/config-ui/styles/globals.css
@@ -2,8 +2,9 @@
body {
padding: 0;
margin: 0;
- font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen,
- Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
+ /* font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen,
+ Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif; */
+ font-family: 'Source Sans Pro', sans-serif;
}
a {
@@ -13,4 +14,24 @@
* {
box-sizing: border-box;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+
+h1, h2 {
+ font-family: 'Rubik', sans-serif;
+}
+
+h1 {
+ color: #303033;
+ font-weight: 600;
+}
+
+h2 {
+ color: #4B4D58;
+ font-weight: 500;
+}
+
+p {
+ color: #717484;
}
diff --git a/config-ui/ui.Dockerfile b/config-ui/ui.Dockerfile
deleted file mode 100644
index 93f2ab9..0000000
--- a/config-ui/ui.Dockerfile
+++ /dev/null
@@ -1,13 +0,0 @@
-FROM node:12-alpine
-
-RUN mkdir -p /usr/src/app/config-ui
-
-WORKDIR /usr/src/app/config-ui
-
-COPY ./config-ui/package.json /usr/src/app/config-ui
-
-RUN yarn
-
-EXPOSE 4000
-
-CMD [ "yarn", "dev" ]
diff --git a/config-ui/utils/envValue.js b/config-ui/utils/envValue.js
index 7416428..1392ad8 100644
--- a/config-ui/utils/envValue.js
+++ b/config-ui/utils/envValue.js
@@ -2,7 +2,7 @@
import os from 'os'
import fs from 'fs'
-const envFilePath = path.join(process.cwd(), 'data', '../../config-ui/.env')
+const envFilePath = process.env.ENV_FILEPATH || path.join(process.cwd(), 'data', '../../.env')
// read .env file
const readEnvVars = () => fs.readFileSync(envFilePath, "utf-8").split(os.EOL)
diff --git a/config/config.go b/config/config.go
index a0d417f..e5d18ad 100644
--- a/config/config.go
+++ b/config/config.go
@@ -3,12 +3,42 @@
import (
_ "github.com/joho/godotenv/autoload"
"github.com/spf13/viper"
+ "os"
+ "path"
)
var V *viper.Viper
func init() {
V = viper.New()
+ configFile := os.Getenv("ENV_FILE")
+ if configFile != "" {
+ if !path.IsAbs(configFile) {
+ panic("Please set ENV_FILE with absolute path. " +
+ "Currently it should only be used for go test to load ENVs.")
+ }
+ V.SetConfigFile(configFile)
+ V.Set("WORKING_DIRECTORY", path.Dir(configFile))
+ } else {
+ V.SetConfigName(".env")
+ V.SetConfigType("env")
+
+ V.AddConfigPath(".")
+ V.AddConfigPath("conf")
+ V.AddConfigPath("etc")
+
+ execPath, execErr := os.Executable()
+ if execErr == nil {
+ V.AddConfigPath(path.Dir(execPath))
+ V.AddConfigPath(path.Join(path.Dir(execPath), "conf"))
+ V.AddConfigPath(path.Join(path.Dir(execPath), "etc"))
+ }
+
+ wdPath, _ := os.Getwd()
+ V.Set("WORKING_DIRECTORY", wdPath)
+ }
+
+ _ = V.ReadInConfig()
V.AutomaticEnv()
V.SetDefault("PORT", ":8080")
}
diff --git a/devops/lake-builder/Dockerfile b/devops/lake-builder/Dockerfile
index b038895..6ee5ef8 100644
--- a/devops/lake-builder/Dockerfile
+++ b/devops/lake-builder/Dockerfile
@@ -2,3 +2,4 @@
RUN apk update
RUN apk upgrade
RUN apk add --update gcc=10.2.1_pre1-r3 g++=10.2.1_pre1-r3
+RUN apk add --no-cache tzdata
diff --git a/devops/lake-builder/README.md b/devops/lake-builder/README.md
index 7e32683..2e49260 100644
--- a/devops/lake-builder/README.md
+++ b/devops/lake-builder/README.md
@@ -5,7 +5,7 @@
## release
```shell
-export VERSION=0.0.1
+export VERSION=0.0.2
docker build -t mericodev/lake-builder:$VERSION .
docker push mericodev/lake-builder:$VERSION
```
diff --git a/docker-compose.yml b/docker-compose.yml
index cb6ca7f..398b09f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -29,17 +29,6 @@
restart: always
depends_on:
- mysql
- config-ui:
- build:
- context: .
- dockerfile: ./config-ui/ui.Dockerfile
- ports:
- - "4000:4000"
- volumes:
- - ./config-ui:/usr/src/app/config-ui
- - /usr/src/app/config-ui/node_modules
- - /usr/src/app/config-ui/.next
- - ./.env:/usr/src/app/config-ui/.env
devlake:
image: mericodev/lake:latest
ports:
@@ -49,6 +38,14 @@
restart: always
depends_on:
- grafana
+ config-ui:
+ image: mericodev/config-ui:latest
+ ports:
+ - 127.0.0.1:4000:4000
+ environment:
+ - ENV_FILEPATH=/.env
+ volumes:
+ - ./.env:/.env:rw
volumes:
mysql-storage:
grafana-storage:
diff --git a/grafana/grafana.config b/grafana/grafana.config
index eb3af6b..678c632 100644
--- a/grafana/grafana.config
+++ b/grafana/grafana.config
@@ -1,4 +1,5 @@
# Config Options
GF_USERS_ALLOW_SIGN_UP=false
GF_DASHBOARDS_JSON_ENABLED=true
-GF_INSTALL_PLUGINS=grafana-piechart-panel
\ No newline at end of file
+GF_INSTALL_PLUGINS=grafana-piechart-panel
+GF_LIVE_ALLOWED_ORIGINS=http://localhost:3002
diff --git a/main.go b/main.go
index e406527..f3a3e6b 100644
--- a/main.go
+++ b/main.go
@@ -6,7 +6,7 @@
)
func main() {
- err := plugins.LoadPlugins("./plugins")
+ err := plugins.LoadPlugins(plugins.PluginDir())
if err != nil {
panic(err)
}
diff --git a/plugins/HOW-TO-ADD-METRICS.md b/plugins/HOW-TO-ADD-METRICS.md
index 8e924ca..d091d55 100644
--- a/plugins/HOW-TO-ADD-METRICS.md
+++ b/plugins/HOW-TO-ADD-METRICS.md
@@ -3,6 +3,24 @@
You love getting data from a source like GitLab, but perhaps you're not getting the the metric you want. Good news! You can get any metric you like by contributing a little bit of code.
-Each plugin has an Enricher. Just navigate into the plugin folder to find the enrichment folder. There, you should find all the various enrichment methods. The job of the Enricher is to grab the data that was collected by the Collector, augment it, and add it to a new DB in accordance with a new schema (which you will have to create).
+1. Decide if you are going to need new data for your metric, or if you just need to enrich the data that already exists.
-- [ ] Details of how to add metrics with new go setup
+2. If you need new data, then you will need to:
+ a. Create a model for the data you'd like to capture.
+ b. Add your new model to the plugin's init file. This will allow GORM to
+ auto-migrate the definition to create a new DB table.
+ c. Create a collector in the "tasks" folder. You will need to make API calls to gather and save your new data.
+ You may need to do some research to figure out what fields are returned from the API to specifically capture
+ everything you would like to calculate your metric.
+ d. Add any additional "enrichment" calculations on top of the data that you are fetching.
+ e. Add your "collection" method to the plugin's execute function in the main package.
+ f. Start the project, trigger an API request that triggers your plugin, and watch the data flow in!
+ g. Congrats, you are done!
+
+3. If you do not need new data then all you need to do is add any additional "enrichment" calculations on top of the data that
+ we are already are fetching!
+
+Thanks for your contribution!
+
+-- The Dev Lake Team
+
diff --git a/plugins/README.md b/plugins/README.md
index 9e01049..cedb97c 100644
--- a/plugins/README.md
+++ b/plugins/README.md
@@ -1,11 +1,103 @@
-# ⚠️ (WIP) So you want to Build a New Plugin...
+# So you want to Build a New Plugin...
...the good news is, it's easy!
-- [ ] Make new plugin doc with new go setup
- - Basic Interface
- - Summary
- - The Collector
- - The Enricher
- - Step by Step
+## Basic Interface
+
+```golang
+type YourPlugin string
+
+func (plugin YourPlugin) Description() string {
+ return "To collect and enrich data from YourPlugin"
+}
+
+func (plugin YourPlugin) Execute(options map[string]interface{}, progress chan<- float32) {
+ logger.Print("Starting YourPlugin execution...")
+
+ // Check fields that are needed in options.
+ projectId, ok := options["projectId"]
+ if !ok {
+ logger.Print("projectId is required for YourPlugin execution")
+ return
+ }
+
+ // Start collecting stuff.
+ if err := tasks.CollectProject(projectId); err != nil {
+ logger.Error("Could not collect projects: ", err)
+ return
+ }
+ // Handle error.
+ if err != nil {
+ logger.Error(err)
+ }
+
+ // Export a variable named PluginEntry for Framework to search and load
+ var PluginEntry YourPlugin //nolint
+}
+```
+
+## Summary
+
+ To build a new plugin you will need a few things. You should choose an API that you'd like to see data from. Think about the metrics you would like to see first, and then look for data that can support those metrics.
+
+## Collection
+
+ Then you will want to write a collection to gather data. You will need to do some reading of the API documentation to figure out what metrics you will want to see at the end in your Grafana dashboard (configuring Grafana is the final step).
+
+## Build a Fetcher to make Requests
+
+The plugins/core folder contains an api client that you can implement within your own plugin. It has useful methods like Get().
+Each API handles pagination differently, so you will likely need to implement a "fetch with pagination" method. One way to do
+this is to use the "ants" package as a way to manage tasks concurrently: https://github.com/panjf2000/ants
+
+Your colection methods may look something like this:
+
+```golang
+func Collect() error {
+ pluginApiClient := CreateApiClient()
+
+ return pluginApiClient.FetchWithPagination("<your_api_url>",
+ func(res *http.Response) error {
+ pluginApiResponse := &ApiResponse{}
+ // You must unmarshal the response from the api to use the results.
+ err := core.UnmarshalResponse(res, pluginApiResponse)
+ if err != nil {
+ logger.Error("Error: ", err)
+ return nil
+ }
+ // loop through the results and save them to the database.
+ for _, value := range *pluginApiResponse {
+ pluginModel := &models.pluginModel{
+ pluginId: value.pluginId,
+ Title: value.Title,
+ Message: value.Message,
+ }
+
+ err = lakeModels.Db.Clauses(clause.OnConflict{
+ UpdateAll: true,
+ }).Create(&pluginModel).Error
+
+ if err != nil {
+ logger.Error("Could not upsert: ", err)
+ }
+ }
+
+ return nil
+ })
+}
+```
+
+Note the use of "upsert". This is useful for only saving modified records.
+
+## Enrichment
+
+ Once you have collected data from the API, you may want to enrich that data by:
+
+ - Add fields you don't currently have
+ - Compute fields you might want for metrics
+ - Eliminate fields you don't need
+
+## You're Done!
+
+Congratulations! You have created your first plugin! 🎖
\ No newline at end of file
diff --git a/plugins/gitlab/README.md b/plugins/gitlab/README.md
index ea99051..23c1140 100644
--- a/plugins/gitlab/README.md
+++ b/plugins/gitlab/README.md
@@ -7,11 +7,12 @@
Pull Request Count | Number of Pull/Merge Requests
Pull Request Pass Rate | Ratio of Pull/Merge Review requests to merged
Pull Request Reviewer Count | Number of Pull/Merge Reviewers
-Pull Request Review Time | Time from the first Pull/Merge Review comment until merged
+Pull Request Review Time | Time from Pull/Merge created time until merged
Commit Author Count | Number of Contributors
Commit Count | Number of Commits
Added Lines | Accumulated Number of New Lines
Deleted Lines | Accumulated Number of Removed Lines
+Pull Request Review Rounds | Number of cycles of commits followed by comments/final merge
## Configuration
diff --git a/plugins/jira/README.md b/plugins/jira/README.md
index 95cae42..846adbb 100644
--- a/plugins/jira/README.md
+++ b/plugins/jira/README.md
@@ -23,7 +23,9 @@
## Configuration
-Set following Environment Variables before launching:
+Set the following environment variables in `.env` file before launching.
+For what's issue status mapping, see [Issue status mapping](#issue-status-mapping) section.
+For what's issue type mapping, see [Issue type mapping](#issue-type-mapping) section.
```sh
######################
@@ -61,6 +63,39 @@
JIRA_ISSUE_STORYPOINT_FIELD=customfield_10024
```
+
+## Issue status mapping<a id="issue-status-mapping"></a>
+Jira is highly customizable, different company may use different `status name` to represent whether a issue was
+resolved or not, one may named it "Done" and others might named it "Finished".
+In order to collect life-cycle information correctly, you'll have to map your specific status to Devlake's standard
+status, Devlake supports two standard status:
+
+ - `Resolved`: issue was ended successfully
+ - `Rejected`: issue was ended by termination or cancellation
+
+Say we were using `Done` and `Cancelled` to represent the final stage of `Story` issues, what we have to do is setting
+the following `Environment Variables` before running Devlake:
+```sh
+JIRA_ISSUE_STORY_STATUS_MAPPING=Resolved:Done;Reject:Cancelled
+```
+
+
+## Issue type mapping<a id="issue-type-mapping"></a>
+Same as status mapping, different company might use different issue type to represent their Bug/Incident/Requirement,
+type mapping is for Devlake to recognize your specific setup.
+Devlake supports three different standard types:
+
+ - `Bug`
+ - `Incident`
+ - `Requirement`
+
+Say we were using `Story` to represent our Requirement, what we have to do is setting the following
+`Environment Variables` before running Devlake:
+```sh
+JIRA_ISSUE_TYPE_MAPPING=Requirement:Story
+```
+
+
## Find Board Id
1. Navigate to the Jira board in the browser
2. in the URL bar, get the board id from the parameter `?rapidView=`
@@ -82,4 +117,6 @@
Using URL
1. Navigate to Administration >> Issues >> Custom Fields .
2. Click the cog and hover over Configure or Screens option.
-3. Observe the URL at the bottom left of the browser window. Example: The id for this custom field is 10006.
\ No newline at end of file
+3. Observe the URL at the bottom left of the browser window. Example: The id for this custom field is 10006.
+
+
diff --git a/plugins/plugins.go b/plugins/plugins.go
index 680b50a..d04266a 100644
--- a/plugins/plugins.go
+++ b/plugins/plugins.go
@@ -3,7 +3,8 @@
import (
"errors"
"fmt"
- "io/ioutil"
+ "github.com/merico-dev/lake/config"
+ "io/fs"
"path"
"path/filepath"
"plugin"
@@ -14,35 +15,23 @@
. "github.com/merico-dev/lake/plugins/core"
)
-// store all plugins
+// Plugins store all plugins
var Plugins map[string]Plugin
-// load plugins from local directory
+// LoadPlugins load plugins from local directory
func LoadPlugins(pluginsDir string) error {
Plugins = make(map[string]Plugin)
- dirs, err := ioutil.ReadDir(pluginsDir)
- if err != nil {
- return err
- }
- for _, subDir := range dirs {
- if !subDir.IsDir() {
- continue
- }
- subDirPath := path.Join(pluginsDir, subDir.Name())
- files, err := ioutil.ReadDir(subDirPath)
+ walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
- for _, file := range files {
- if file.IsDir() || !strings.HasSuffix(file.Name(), ".so") {
- continue
- }
- logger.Info(`[plugin-core] find a plugin`, file.Name())
- so := filepath.Join(subDirPath, file.Name())
- plug, err := plugin.Open(so)
- logger.Info(`[plugin-core] open a plugin success`, file.Name())
- if err != nil {
- return err
+ fileName := d.Name()
+ println(fileName, path)
+ if strings.HasSuffix(fileName, ".so") {
+ pluginName := fileName[0:len(d.Name())-3]
+ plug, loadErr := plugin.Open(path)
+ if loadErr != nil {
+ return loadErr
}
symPluginEntry, pluginEntryError := plug.Lookup("PluginEntry")
if pluginEntryError != nil {
@@ -50,27 +39,35 @@
}
plugEntry, ok := symPluginEntry.(Plugin)
if !ok {
- return fmt.Errorf("%v PluginEntry must implement Plugin interface", file.Name())
+ return fmt.Errorf("%v PluginEntry must implement Plugin interface", pluginName)
}
plugEntry.Init()
- logger.Info(`[plugin-core] init a plugin success`, file.Name())
- Plugins[subDir.Name()] = plugEntry
-
- logger.Info("[plugin-core] plugin loaded", subDir.Name())
- break
+ logger.Info(`[plugin-core] init a plugin success`, pluginName)
+ Plugins[pluginName] = plugEntry
+ logger.Info("[plugin-core] plugin loaded", pluginName)
}
- }
- return nil
+ return nil
+ })
+ return walkErr
}
func RunPlugin(name string, options map[string]interface{}, progress chan<- float32) error {
if Plugins == nil {
return errors.New("plugins have to be loaded first, please call LoadPlugins beforehand")
}
- plugin, ok := Plugins[name]
+ p, ok := Plugins[name]
if !ok {
return fmt.Errorf("unable to find plugin with name %v", name)
}
- plugin.Execute(options, progress)
+ p.Execute(options, progress)
return nil
}
+
+func PluginDir() string {
+ pluginDir := config.V.GetString("PLUGIN_DIR")
+ if !path.IsAbs(pluginDir) {
+ wd := config.V.GetString("WORKING_DIRECTORY")
+ pluginDir = filepath.Join(wd, pluginDir)
+ }
+ return pluginDir
+}
\ No newline at end of file
diff --git a/scripts/compile-plugins.sh b/scripts/compile-plugins.sh
index 9b4c60f..38404c3 100755
--- a/scripts/compile-plugins.sh
+++ b/scripts/compile-plugins.sh
@@ -1,6 +1,12 @@
#!/bin/sh
-for PLUG in $(find plugins/* -maxdepth 0 -type d -not -name core -not -empty); do
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+PLUGIN_SRC_DIR=$SCRIPT_DIR/../plugins
+PLUGIN_OUTPUT_DIR=$SCRIPT_DIR/../bin/plugins
+
+
+for PLUG in $(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -empty); do
NAME=$(basename $PLUG)
- echo $PLUG/$NAME
- go build -buildmode=plugin "$@" -o $PLUG/$NAME.so $PLUG/*.go
+ echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so"
+ go build -buildmode=plugin "$@" -o $PLUGIN_OUTPUT_DIR/$NAME/$NAME.so $PLUG/*.go
done
diff --git a/scripts/dev.sh b/scripts/dev.sh
deleted file mode 100755
index d5d1dcc..0000000
--- a/scripts/dev.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-source ./scripts/compile-plugins.sh
-source ./scripts/export-env.sh
-go build -o lake
-./lake
diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh
deleted file mode 100644
index f83edfd..0000000
--- a/scripts/e2e-test.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-source ./scripts/compile-plugins.sh
-source ./scripts/export-env.sh
-go test -v ./test/...
diff --git a/scripts/export-env.sh b/scripts/export-env.sh
index c8bb09c..f5c088b 100755
--- a/scripts/export-env.sh
+++ b/scripts/export-env.sh
@@ -1,7 +1,10 @@
#!/bin/sh
-FILE=.env
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+FILE=$SCRIPT_DIR/../.env
+
if [ -f "$FILE" ]; then
- export $(grep -v '^#' .env | sed 's/#.*$//g' | xargs)
+ export $(grep -v '^#' $FILE | sed 's/#.*$//g' | xargs)
else
echo "$FILE does not exist."
fi
diff --git a/scripts/pm.sh b/scripts/pm.sh
index 54b3b37..2c2c1ba 100755
--- a/scripts/pm.sh
+++ b/scripts/pm.sh
@@ -2,17 +2,19 @@
set -e
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+
LAKE_ENDPOINT=${LAKE_ENDPOINT-'http://localhost:8080'}
LAKE_TASK_URL=$LAKE_ENDPOINT/task
debug() {
- scripts/compile-plugins.sh -gcflags=all="-N -l"
+ $SCRIPT_DIR/compile-plugins.sh -gcflags=all="-N -l"
dlv debug
}
run() {
- scripts/compile-plugins.sh
- go run main.go
+ $SCRIPT_DIR/compile-plugins.sh
+ go run $SCRIPT_DIR/../main.go
}
jira() {
diff --git a/scripts/unit-test.sh b/scripts/unit-test.sh
deleted file mode 100755
index 682ebb2..0000000
--- a/scripts/unit-test.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-set -e
-./scripts/export-env.sh
-./scripts/compile-plugins.sh
-go test -v $(go list ./... | grep -v /test/)
diff --git a/test/plugins/plugins_test.go b/test/plugins/plugins_test.go
index 5b97901..988b24f 100644
--- a/test/plugins/plugins_test.go
+++ b/test/plugins/plugins_test.go
@@ -1,18 +1,16 @@
package plugins
import (
+ "github.com/magiconair/properties/assert"
+ "github.com/merico-dev/lake/config"
+ "github.com/merico-dev/lake/plugins"
"path"
- "runtime"
"strings"
"testing"
-
- "github.com/merico-dev/lake/plugins"
)
func TestPluginsLoading(t *testing.T) {
- _, filename, _, _ := runtime.Caller(0)
- pluginsDir := strings.Replace(path.Dir(filename), `test/`, ``, 1)
- err := plugins.LoadPlugins(pluginsDir)
+ err := plugins.LoadPlugins(plugins.PluginDir())
if err != nil {
t.Errorf("Failed to LoadPlugins %v", err)
}
@@ -38,3 +36,9 @@
// }
}
+
+func TestPluginDir(t *testing.T) {
+ pluginDir := plugins.PluginDir()
+ assert.Equal(t, path.IsAbs(pluginDir), true)
+ assert.Equal(t, strings.HasSuffix(pluginDir, config.V.GetString("PLUGIN_DIR")), true)
+}
diff --git a/zoneinfo.zip b/zoneinfo.zip
deleted file mode 100644
index d32fbba..0000000
--- a/zoneinfo.zip
+++ /dev/null
Binary files differ
