commit	68b51d3d668f9e5bbc34f6a0537d10b9b069fe3e	[log] [tgz]
author	baunsgaard <baunsgaard@tugraz.at>	Mon Aug 24 11:04:09 2020 +0200
committer	baunsgaard <baunsgaard@tugraz.at>	Mon Aug 24 13:30:21 2020 +0200
tree	05f979b91e17278909e62bd0442f877646ef1839
parent	c0643923206616b7503d8b74884b3e8d40ccfae3 [diff]

[SYSTEMDS-2638] Federated stats for Worker

Federated workers now are able to output stats of execution, if the -stats
flag is set. The stats are printed at every clear command sendt to the
worker. Each time the stats are printed the stats are also reset.

This commit also change the test setup to worker now in separated JVM in
test frame work. To enable this the federated worker startup wait time
have been increased to 1 sec from 0.75 sec

.github/workflows/functionsTests.yml[diff]
src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java[diff]
src/main/java/org/apache/sysds/utils/Statistics.java[diff]
src/test/java/org/apache/sysds/test/AutomatedTestBase.java[diff]
src/test/java/org/apache/sysds/test/TestUtils.java[diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedBivarTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedBivarTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedGLMTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedGLMTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedKmeansTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedKmeansTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedL2SVMTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedL2SVMTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedLogRegTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedLogRegTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedPCATest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedPCATest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedUnivarTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedUnivarTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedYL2SVMTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedYL2SVMTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedBinaryMatrixTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedBinaryMatrixTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedBinaryVectorTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedBinaryVectorTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedConstructionTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedConstructionTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedMatrixScalarOperationsTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedMatrixScalarOperationsTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedMultiplyTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedMultiplyTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedNegativeTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedNegativeTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRCBindTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedRCBindTest.java - diff]
src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedSumTest.java[Renamed from src/test/java/org/apache/sysds/test/functions/federated/FederatedSumTest.java - diff]
src/test/java/org/apache/sysds/test/functions/privacy/FederatedL2SVMTest.java[diff]
src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java[diff]
src/test/java/org/apache/sysds/test/functions/transform/TransformFederatedEncodeDecodeTest.java[diff]

24 files changed

tree: 05f979b91e17278909e62bd0442f877646ef1839

README.md

Apache SystemDS

Overview: SystemDS is a versatile system for the end-to-end data science lifecycle from data integration, cleaning, and feature engineering, over efficient, local and distributed ML model training, to deployment and serving. To this end, we aim to provide a stack of declarative languages with R-like syntax for (1) the different tasks of the data-science lifecycle, and (2) users with different expertise. These high-level scripts are compiled into hybrid execution plans of local, in-memory CPU and GPU operations, as well as distributed operations on Apache Spark. In contrast to existing systems - that either provide homogeneous tensors or 2D Datasets - and in order to serve the entire data science lifecycle, the underlying data model are DataTensors, i.e., tensors (multi-dimensional arrays) whose first dimension may have a heterogeneous and nested schema.

Quick Start Install, Quick Start and Hello World

Documentation: SystemDS Documentation

Python Documentation Python SystemDS Documentation

Issue Tracker Jira Dashboard

Status and Build: SystemDS is still in pre-alpha status. The original code base was forked from Apache SystemML 1.2 in September 2018. We will continue to support linear algebra programs over matrices, while replacing the underlying data model and compiler, as well as substantially extending the supported functionalities. Until the first release, you can build your own snapshot via Apache Maven: mvn clean package -P distribution.