commit	64e0573aca01dfd07b23ec41e20acb307829733e	[log] [tgz]
author	Vinoth Chandar <vinoth@uber.com>	Mon Aug 28 01:28:08 2017 -0700
committer	vinoth chandar <vinothchandar@users.noreply.github.com>	Mon Oct 02 20:44:53 2017 -0700
tree	7cb72248968ce86dbb7057d906668fe93014ed6c
parent	c98ee057fcd9d2566e5cefcf15bd5d2e5ec9283e [diff]

Adding hoodie-spark to support Spark Datasource for Hoodie

 - Write with COW/MOR paths work fully
 - Read with RO view works on both storages*
 - Incremental view supported on COW
 - Refactored out HoodieReadClient methods, to just contain key based access
 - HoodieDataSourceHelpers class can be now used to construct inputs to datasource
 - Tests in hoodie-client using new helpers and mechanisms
 - Basic tests around save modes & insert/upserts (more to follow)
 - Bumped up scala to 2.11, since 2.10 is deprecated & complains with scalatest
 - Updated documentation to describe usage
 - New sample app written using the DataSource API

docs/_data/sidebars/mydoc_sidebar.yml[diff]
docs/configurations.md[diff]
docs/incremental_processing.md[diff]
docs/quickstart.md[diff]
hoodie-cli/pom.xml[diff]
hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HDFSParquetImportCommand.java[diff]
hoodie-client/pom.xml[diff]
hoodie-client/src/main/java/com/uber/hoodie/HoodieReadClient.java[diff]
hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java[diff]
hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java[diff]
hoodie-client/src/test/java/HoodieClientExample.java[diff]
hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java[Renamed from hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java - diff]
hoodie-client/src/test/java/com/uber/hoodie/common/HoodieClientTestUtils.java[diff]
hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java[diff]
hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java[Renamed from hoodie-client/src/test/java/com/uber/hoodie/TestMergeOnReadTable.java - diff]
hoodie-common/src/main/java/com/uber/hoodie/common/util/ParquetUtils.java[diff]
hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieROTablePathFilter.java[diff]
hoodie-spark/pom.xml[Added - diff]
hoodie-spark/src/main/java/com/uber/hoodie/BaseAvroPayload.java[Renamed from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/DeltaStreamerPayload.java - diff]
hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java[Added - diff]
hoodie-spark/src/main/java/com/uber/hoodie/HoodieDataSourceHelpers.java[Added - diff]
hoodie-spark/src/main/java/com/uber/hoodie/KeyGenerator.java[Renamed from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/KeyGenerator.java - diff]
hoodie-spark/src/main/java/com/uber/hoodie/OverwriteWithLatestAvroPayload.java[Renamed from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/DeltaStreamerAvroPayload.java - diff]
hoodie-spark/src/main/java/com/uber/hoodie/SimpleKeyGenerator.java[Added - diff]
hoodie-spark/src/main/scala/com/uber/hoodie/AvroConversionUtils.scala[Added - diff]
hoodie-spark/src/main/scala/com/uber/hoodie/DataSourceOptions.scala[Added - diff]
hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala[Added - diff]
hoodie-spark/src/main/scala/com/uber/hoodie/IncrementalRelation.scala[Added - diff]
hoodie-spark/src/main/scala/com/uber/hoodie/package.scala[Added - diff]
hoodie-spark/src/test/java/DataSourceTestUtils.java[Added - diff]
hoodie-spark/src/test/java/HoodieJavaApp.java[Added - diff]
hoodie-spark/src/test/resources/log4j-surefire.properties[Added - diff]
hoodie-spark/src/test/scala/DataSourceTest.scala[Added - diff]
hoodie-utilities/pom.xml[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/UtilHelpers.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/SimpleKeyGenerator.java[Deleted - diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/TimestampBasedKeyGenerator.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/FilebasedSchemaProvider.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/DFSSource.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HiveIncrPullSource.java[diff]
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/KafkaSource.java[diff]
hoodie-utilities/src/main/resources/delta-streamer-config/key-generator.properties[diff]
hoodie-utilities/src/test/java/HoodieSparkSQLExample.java[Deleted - diff]
pom.xml[diff]

45 files changed

tree: 7cb72248968ce86dbb7057d906668fe93014ed6c

README.md

Hoodie

Hoodie manages storage of large analytical datasets on HDFS and serve them out via two types of tables

Read Optimized Table - Provides excellent query performance via purely columnar storage (e.g. Parquet)
Near-Real time Table (WIP) - Provides queries on real-time data, using a combination of columnar & row based storage (e.g Parquet + Avro)

For more, head over here